@inflexa-ai/harness 0.39.1 → 0.39.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/execution/artifact-registration.d.ts +12 -2
- package/dist/execution/artifact-registration.js +9 -4
- package/dist/execution/post-step-pipeline.d.ts +3 -6
- package/dist/execution/post-step-pipeline.js +11 -3
- package/dist/index.d.ts +2 -2
- package/dist/index.js +1 -1
- package/dist/report-model/pin-snapshot.js +4 -10
- package/dist/state/artifacts.d.ts +11 -9
- package/dist/state/artifacts.js +112 -93
- package/dist/state/data-profile.d.ts +2 -2
- package/dist/state/data-profile.js +2 -2
- package/dist/tasks/data-profile.d.ts +28 -10
- package/dist/tasks/data-profile.js +114 -95
- package/dist/workflows/execute-analysis.js +5 -3
- package/package.json +1 -1
|
@@ -12,6 +12,7 @@
|
|
|
12
12
|
import { type Result } from "neverthrow";
|
|
13
13
|
import type { Pool, PoolClient } from "pg";
|
|
14
14
|
import type { AgentSession } from "../auth/types.js";
|
|
15
|
+
import type { DbError } from "../lib/db-result.js";
|
|
15
16
|
import { type GateRefusal } from "../lib/hooks.js";
|
|
16
17
|
import type { Logger } from "../lib/logger.js";
|
|
17
18
|
import type { ArtifactRegistry, ArtifactRegistrationInput } from "./artifact-registry.js";
|
|
@@ -29,6 +30,14 @@ export interface ArtifactRegistrationResult {
|
|
|
29
30
|
error: string;
|
|
30
31
|
}>;
|
|
31
32
|
}
|
|
33
|
+
/** Why a registration stopped: the `register` gate refused it, or a write of the local ledger failed. */
|
|
34
|
+
export type ArtifactRegistrationFailure = {
|
|
35
|
+
readonly kind: "refused";
|
|
36
|
+
readonly refusal: GateRefusal;
|
|
37
|
+
} | {
|
|
38
|
+
readonly kind: "ledger_failed";
|
|
39
|
+
readonly error: DbError;
|
|
40
|
+
};
|
|
32
41
|
/**
|
|
33
42
|
* Register step-level artifacts (paths under `runs/{runId}/{stepId}/`).
|
|
34
43
|
*
|
|
@@ -37,6 +46,7 @@ export interface ArtifactRegistrationResult {
|
|
|
37
46
|
* and applies any returned external ids back onto the local rows.
|
|
38
47
|
*
|
|
39
48
|
* A refusal of `register` writes no external id: the upserted rows keep
|
|
40
|
-
* `artifact_id = NULL`.
|
|
49
|
+
* `artifact_id = NULL`. A failed ledger write stops the registration at that
|
|
50
|
+
* write.
|
|
41
51
|
*/
|
|
42
|
-
export declare function registerStepArtifacts(db: Pool | PoolClient, registry: ArtifactRegistry, input: ArtifactRegistrationInput, session: AgentSession, logger?: Logger): Promise<Result<ArtifactRegistrationResult,
|
|
52
|
+
export declare function registerStepArtifacts(db: Pool | PoolClient, registry: ArtifactRegistry, input: ArtifactRegistrationInput, session: AgentSession, logger?: Logger): Promise<Result<ArtifactRegistrationResult, ArtifactRegistrationFailure>>;
|
|
@@ -22,7 +22,8 @@ import { upsertArtifacts, updateArtifactId } from "../state/index.js";
|
|
|
22
22
|
* and applies any returned external ids back onto the local rows.
|
|
23
23
|
*
|
|
24
24
|
* A refusal of `register` writes no external id: the upserted rows keep
|
|
25
|
-
* `artifact_id = NULL`.
|
|
25
|
+
* `artifact_id = NULL`. A failed ledger write stops the registration at that
|
|
26
|
+
* write.
|
|
26
27
|
*/
|
|
27
28
|
export async function registerStepArtifacts(db, registry, input, session,
|
|
28
29
|
// A parameter rather than a field on `ArtifactRegistrationInput`: that type is
|
|
@@ -51,11 +52,13 @@ logger = createNoopLogger()) {
|
|
|
51
52
|
fileType: a.type,
|
|
52
53
|
};
|
|
53
54
|
});
|
|
54
|
-
await upsertArtifacts(db, localEntries);
|
|
55
|
+
const upserted = await upsertArtifacts(db, localEntries);
|
|
56
|
+
if (upserted.isErr())
|
|
57
|
+
return err({ kind: "ledger_failed", error: upserted.error });
|
|
55
58
|
const localPaths = new Set(localEntries.map((e) => e.path));
|
|
56
59
|
const registered = await passGate("ArtifactRegistry.register", registry.register(input, session));
|
|
57
60
|
if (registered.isErr())
|
|
58
|
-
return err(registered.error);
|
|
61
|
+
return err({ kind: "refused", refusal: registered.error });
|
|
59
62
|
const result = registered.value;
|
|
60
63
|
// The registry excluded these from `failed` by its own severity judgement, so
|
|
61
64
|
// this is their only record. Logged here rather than left to the registry:
|
|
@@ -72,7 +75,9 @@ logger = createNoopLogger()) {
|
|
|
72
75
|
// (e.g., data inputs that were already born-synced).
|
|
73
76
|
if (!localPaths.has(reg.path))
|
|
74
77
|
continue;
|
|
75
|
-
await updateArtifactId(db, resourceId, reg.path, reg.externalId);
|
|
78
|
+
const updated = await updateArtifactId(db, resourceId, reg.path, reg.externalId);
|
|
79
|
+
if (updated.isErr())
|
|
80
|
+
return err({ kind: "ledger_failed", error: updated.error });
|
|
76
81
|
externalRegistered++;
|
|
77
82
|
}
|
|
78
83
|
return ok({
|
|
@@ -17,10 +17,10 @@ import type { WorkspaceFilesystem } from "../workspace/filesystem.js";
|
|
|
17
17
|
import type { ResolveWorkspaceRoot } from "../workspace/paths.js";
|
|
18
18
|
import type { ArtifactManifestEntry } from "../schemas/artifact-manifest.js";
|
|
19
19
|
import type { StepSummary } from "../schemas/step-summary.js";
|
|
20
|
-
import type { GateRefusal } from "../lib/hooks.js";
|
|
21
20
|
import type { Logger } from "../lib/logger.js";
|
|
22
21
|
import type { UsageRecorder } from "../billing/usage-recorder.js";
|
|
23
22
|
import type { ArtifactRegistry } from "./artifact-registry.js";
|
|
23
|
+
import { type ArtifactRegistrationFailure } from "./artifact-registration.js";
|
|
24
24
|
import { type FileMetadataEntry } from "./artifact-metadata.js";
|
|
25
25
|
import type { PostStepArtifacts, PostStepContext, StepOutputs } from "../workflows/sandbox-step.js";
|
|
26
26
|
/**
|
|
@@ -55,10 +55,7 @@ export declare function generateStepFileMetadata(deps: PostStepPipelineDeps, pos
|
|
|
55
55
|
* still returned for vector indexing.
|
|
56
56
|
*/
|
|
57
57
|
export declare function generateStepSummaryAndWrite(deps: PostStepPipelineDeps, postCtx: PostStepContext, manifest: readonly ArtifactManifestEntry[]): Promise<StepSummary | undefined>;
|
|
58
|
-
export type StepRegistrationFailure = {
|
|
59
|
-
readonly kind: "refused";
|
|
60
|
-
readonly refusal: GateRefusal;
|
|
61
|
-
} | {
|
|
58
|
+
export type StepRegistrationFailure = ArtifactRegistrationFailure | {
|
|
62
59
|
readonly kind: "rejected";
|
|
63
60
|
readonly message: string;
|
|
64
61
|
};
|
|
@@ -71,7 +68,7 @@ export declare function describeStepRegistrationFailure(failure: StepRegistratio
|
|
|
71
68
|
* plus anything rejected as a consequence of one — so a non-zero count means
|
|
72
69
|
* real outputs went unregistered, and the `err` carries the per-file detail (the
|
|
73
70
|
* OSS filesystem registry returns `externalFailed: 0`, so it never trips). A
|
|
74
|
-
* refusal of the `register` gate
|
|
71
|
+
* refusal of the `register` gate and a failed ledger write are the other `err`s.
|
|
75
72
|
* Returns the reconciled manifest.
|
|
76
73
|
*/
|
|
77
74
|
export declare function reconcileAndRegisterStepArtifacts(deps: PostStepPipelineDeps, postCtx: PostStepContext, manifest: readonly ArtifactManifestEntry[]): Promise<Result<readonly ArtifactManifestEntry[], StepRegistrationFailure>>;
|
|
@@ -14,6 +14,7 @@ import { join } from "node:path";
|
|
|
14
14
|
import { err, ok } from "neverthrow";
|
|
15
15
|
import { unwrapOrThrow } from "../lib/result.js";
|
|
16
16
|
import { createNoopLogger } from "../lib/console-logger.js";
|
|
17
|
+
import { describeDbError } from "../lib/db-result.js";
|
|
17
18
|
import { writeFileWithinRoot } from "../lib/fs-helpers.js";
|
|
18
19
|
import { registerStepArtifacts } from "./artifact-registration.js";
|
|
19
20
|
import { reconcileManifestWithDisk } from "./reconcile-manifest.js";
|
|
@@ -94,7 +95,14 @@ export async function generateStepSummaryAndWrite(deps, postCtx, manifest) {
|
|
|
94
95
|
return summary;
|
|
95
96
|
}
|
|
96
97
|
export function describeStepRegistrationFailure(failure) {
|
|
97
|
-
|
|
98
|
+
switch (failure.kind) {
|
|
99
|
+
case "refused":
|
|
100
|
+
return `the artifact registry refused the registration: ${failure.refusal.reason}`;
|
|
101
|
+
case "ledger_failed":
|
|
102
|
+
return `the artifact ledger write failed: ${describeDbError(failure.error)}`;
|
|
103
|
+
case "rejected":
|
|
104
|
+
return failure.message;
|
|
105
|
+
}
|
|
98
106
|
}
|
|
99
107
|
/**
|
|
100
108
|
* Reconcile the manifest against disk (drop phantoms, rehash) and register the
|
|
@@ -104,7 +112,7 @@ export function describeStepRegistrationFailure(failure) {
|
|
|
104
112
|
* plus anything rejected as a consequence of one — so a non-zero count means
|
|
105
113
|
* real outputs went unregistered, and the `err` carries the per-file detail (the
|
|
106
114
|
* OSS filesystem registry returns `externalFailed: 0`, so it never trips). A
|
|
107
|
-
* refusal of the `register` gate
|
|
115
|
+
* refusal of the `register` gate and a failed ledger write are the other `err`s.
|
|
108
116
|
* Returns the reconciled manifest.
|
|
109
117
|
*/
|
|
110
118
|
export async function reconcileAndRegisterStepArtifacts(deps, postCtx, manifest) {
|
|
@@ -129,7 +137,7 @@ export async function reconcileAndRegisterStepArtifacts(deps, postCtx, manifest)
|
|
|
129
137
|
collector: lineageCollector,
|
|
130
138
|
}, session, deps.logger);
|
|
131
139
|
if (registered.isErr())
|
|
132
|
-
return err(
|
|
140
|
+
return err(registered.error);
|
|
133
141
|
const reg = registered.value;
|
|
134
142
|
if (reg.externalFailed > 0) {
|
|
135
143
|
// The registry commits per leaf and per activity, so accepted and rejected
|
package/dist/index.d.ts
CHANGED
|
@@ -110,8 +110,8 @@ export type { WorkflowStatusString } from "@dbos-inc/dbos-sdk";
|
|
|
110
110
|
export { createPool } from "./lib/storage.js";
|
|
111
111
|
export type { PoolConfig } from "./lib/storage.js";
|
|
112
112
|
export type { Pool } from "pg";
|
|
113
|
-
export { registerDataProfileWorkflow, triggerDataProfile, runDataProfile } from "./tasks/data-profile.js";
|
|
114
|
-
export type { DataProfileDeps, DataProfileWorkflowInput, DataProfileTriggerDeps, DataProfileTriggerParams, DataProfileTriggerResult, } from "./tasks/data-profile.js";
|
|
113
|
+
export { registerDataProfileWorkflow, triggerDataProfile, runDataProfile, describeDataProfileStartError } from "./tasks/data-profile.js";
|
|
114
|
+
export type { DataProfileDeps, DataProfileWorkflowInput, DataProfileTriggerDeps, DataProfileTriggerParams, DataProfileTriggerResult, DataProfileStartError, } from "./tasks/data-profile.js";
|
|
115
115
|
/**
|
|
116
116
|
* The synthetic run id the data-profile workflow stamps onto its own frame — the profile has no run
|
|
117
117
|
* row, so this literal is the only thing that identifies its work as the profile's.
|
package/dist/index.js
CHANGED
|
@@ -210,7 +210,7 @@ export { createPool } from "./lib/storage.js";
|
|
|
210
210
|
// when the analysis names a non-empty seeded input set; a failed profile is re-claimed
|
|
211
211
|
// via `tryRetryDataProfile` and then started with `runDataProfile` (the managed retry
|
|
212
212
|
// route's shape — embedders mirror it).
|
|
213
|
-
export { registerDataProfileWorkflow, triggerDataProfile, runDataProfile } from "./tasks/data-profile.js";
|
|
213
|
+
export { registerDataProfileWorkflow, triggerDataProfile, runDataProfile, describeDataProfileStartError } from "./tasks/data-profile.js";
|
|
214
214
|
/**
|
|
215
215
|
* The synthetic run id the data-profile workflow stamps onto its own frame — the profile has no run
|
|
216
216
|
* row, so this literal is the only thing that identifies its work as the profile's.
|
|
@@ -210,19 +210,13 @@ async function collectCitations(pool, resolveWorkspaceRoot, analysisId) {
|
|
|
210
210
|
* tier, which runs one time for each report version.
|
|
211
211
|
*/
|
|
212
212
|
export async function pinReportSnapshot(pool, analysisId, options = {}) {
|
|
213
|
-
|
|
214
|
-
|
|
215
|
-
|
|
216
|
-
}
|
|
217
|
-
catch (cause) {
|
|
218
|
-
// The query speaks the throw protocol of the `pg` driver. This is the thin wrapper that turns
|
|
219
|
-
// that throw into a value, thus each caller above reads a failure as data.
|
|
220
|
-
return err({ kind: "ledger-read-failed", cause });
|
|
221
|
-
}
|
|
213
|
+
const ledgerRows = await queryAnalysisArtifacts(pool, analysisId);
|
|
214
|
+
if (ledgerRows.isErr())
|
|
215
|
+
return err({ kind: "ledger-read-failed", cause: ledgerRows.error.cause });
|
|
222
216
|
// The ledger accepts any path. A null-prototype map keeps a key such as `__proto__` an ordinary
|
|
223
217
|
// entry, thus no path collides with a prototype slot.
|
|
224
218
|
const artifacts = Object.create(null);
|
|
225
|
-
for (const row of ledgerRows) {
|
|
219
|
+
for (const row of ledgerRows.value) {
|
|
226
220
|
artifacts[row.path] = { hash: row.hash, fileType: row.fileType };
|
|
227
221
|
}
|
|
228
222
|
// A citation reference resolves against this list: the resolver refuses a key that the list does not
|
|
@@ -2,6 +2,8 @@
|
|
|
2
2
|
* `cortex_artifacts` operations — artifact upsert, lookup, and file-id
|
|
3
3
|
* reconciliation.
|
|
4
4
|
*/
|
|
5
|
+
import type { ResultAsync } from "neverthrow";
|
|
6
|
+
import { type DbError } from "../lib/db-result.js";
|
|
5
7
|
import type { Querier } from "./db.js";
|
|
6
8
|
import type { ArtifactRow } from "./schema.js";
|
|
7
9
|
export interface RegisterArtifactInput {
|
|
@@ -20,7 +22,7 @@ export interface RegisterArtifactInput {
|
|
|
20
22
|
*/
|
|
21
23
|
fileId?: string | null;
|
|
22
24
|
}
|
|
23
|
-
export declare function upsertArtifact(pool: Querier, entry: RegisterArtifactInput):
|
|
25
|
+
export declare function upsertArtifact(pool: Querier, entry: RegisterArtifactInput): ResultAsync<void, DbError>;
|
|
24
26
|
/**
|
|
25
27
|
* Batched upsert — writes entries in multi-row `INSERT ... VALUES` statements
|
|
26
28
|
* of at most {@link ROWS_PER_STATEMENT} rows. Callers range from per-step
|
|
@@ -30,7 +32,7 @@ export declare function upsertArtifact(pool: Querier, entry: RegisterArtifactInp
|
|
|
30
32
|
* idempotent per row (`ON CONFLICT ... DO UPDATE`), so a failure between
|
|
31
33
|
* chunks is healed by the caller's retry re-upserting the same manifest.
|
|
32
34
|
*/
|
|
33
|
-
export declare function upsertArtifacts(pool: Querier, entries: RegisterArtifactInput[]):
|
|
35
|
+
export declare function upsertArtifacts(pool: Querier, entries: RegisterArtifactInput[]): ResultAsync<void, DbError>;
|
|
34
36
|
/** Result type for input artifact metadata lookups. */
|
|
35
37
|
export interface InputArtifactMeta {
|
|
36
38
|
path: string;
|
|
@@ -42,8 +44,8 @@ export interface InputArtifactMeta {
|
|
|
42
44
|
* Query cortex_artifacts for input-role rows matching the given paths.
|
|
43
45
|
* Used at the registration boundary to resolve metadata for data-source inputs.
|
|
44
46
|
*/
|
|
45
|
-
export declare function queryInputArtifacts(pool: Querier, analysisId: string, paths: string[]):
|
|
46
|
-
export declare function queryUnsyncedStepArtifacts(pool: Querier, resourceId: string, runId: string, stepId: string):
|
|
47
|
+
export declare function queryInputArtifacts(pool: Querier, analysisId: string, paths: string[]): ResultAsync<InputArtifactMeta[], DbError>;
|
|
48
|
+
export declare function queryUnsyncedStepArtifacts(pool: Querier, resourceId: string, runId: string, stepId: string): ResultAsync<ArtifactRow[], DbError>;
|
|
47
49
|
/** One registered step output, as a downstream consumer needs to name it. */
|
|
48
50
|
export interface StepArtifactRef {
|
|
49
51
|
/** Workspace-root-relative path (`runs/{runId}/{stepId}/output/de.csv`). */
|
|
@@ -56,7 +58,7 @@ export interface StepArtifactRef {
|
|
|
56
58
|
* path, so a `limit` truncates the tail of the list rather than an arbitrary
|
|
57
59
|
* slice of it — and so the same call replays to the same rows.
|
|
58
60
|
*/
|
|
59
|
-
export declare function queryStepArtifactPaths(pool: Querier, resourceId: string, runId: string, stepId: string, limit: number):
|
|
61
|
+
export declare function queryStepArtifactPaths(pool: Querier, resourceId: string, runId: string, stepId: string, limit: number): ResultAsync<StepArtifactRef[], DbError>;
|
|
60
62
|
/** One registered artifact of an analysis, as a reader of the full artifact set names it. */
|
|
61
63
|
export interface AnalysisArtifactRef {
|
|
62
64
|
/** Workspace-root-relative path (`runs/{runId}/{stepId}/output/de.csv`). */
|
|
@@ -72,11 +74,11 @@ export interface AnalysisArtifactRef {
|
|
|
72
74
|
* query drops such a row, a later refusal says that the artifact never existed, and that reason is
|
|
73
75
|
* false.
|
|
74
76
|
*/
|
|
75
|
-
export declare function queryAnalysisArtifacts(pool: Querier, analysisId: string):
|
|
77
|
+
export declare function queryAnalysisArtifacts(pool: Querier, analysisId: string): ResultAsync<AnalysisArtifactRef[], DbError>;
|
|
76
78
|
/** Count step-output artifacts produced by a run — for the run-completed card. */
|
|
77
|
-
export declare function countArtifactsForRun(pool: Querier, analysisId: string, runId: string):
|
|
78
|
-
export declare function updateArtifactId(pool: Querier, resourceId: string, path: string, artifactId: string, fileType?: string | null):
|
|
79
|
+
export declare function countArtifactsForRun(pool: Querier, analysisId: string, runId: string): ResultAsync<number, DbError>;
|
|
80
|
+
export declare function updateArtifactId(pool: Querier, resourceId: string, path: string, artifactId: string, fileType?: string | null): ResultAsync<void, DbError>;
|
|
79
81
|
export declare function updateFileIds(pool: Querier, pairs: Array<{
|
|
80
82
|
artifactId: string;
|
|
81
83
|
fileId: string;
|
|
82
|
-
}>):
|
|
84
|
+
}>): ResultAsync<void, DbError>;
|
package/dist/state/artifacts.js
CHANGED
|
@@ -2,10 +2,12 @@
|
|
|
2
2
|
* `cortex_artifacts` operations — artifact upsert, lookup, and file-id
|
|
3
3
|
* reconciliation.
|
|
4
4
|
*/
|
|
5
|
-
|
|
5
|
+
import { tryMutation, tryQuery } from "../lib/db-result.js";
|
|
6
|
+
export function upsertArtifact(pool, entry) {
|
|
6
7
|
const now = new Date().toISOString();
|
|
7
|
-
|
|
8
|
-
|
|
8
|
+
return tryMutation("artifacts.upsertArtifact", async () => {
|
|
9
|
+
await pool.query({
|
|
10
|
+
text: `INSERT INTO cortex_artifacts
|
|
9
11
|
(analysis_id, path, hash, size, role, source_step, source_run, file_type, file_id, created_at)
|
|
10
12
|
VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10)
|
|
11
13
|
ON CONFLICT (analysis_id, path) DO UPDATE SET
|
|
@@ -16,18 +18,19 @@ export async function upsertArtifact(pool, entry) {
|
|
|
16
18
|
source_run = EXCLUDED.source_run,
|
|
17
19
|
file_type = COALESCE(EXCLUDED.file_type, cortex_artifacts.file_type),
|
|
18
20
|
file_id = COALESCE(EXCLUDED.file_id, cortex_artifacts.file_id)`,
|
|
19
|
-
|
|
20
|
-
|
|
21
|
-
|
|
22
|
-
|
|
23
|
-
|
|
24
|
-
|
|
25
|
-
|
|
26
|
-
|
|
27
|
-
|
|
28
|
-
|
|
29
|
-
|
|
30
|
-
|
|
21
|
+
values: [
|
|
22
|
+
entry.resourceId,
|
|
23
|
+
entry.path,
|
|
24
|
+
entry.hash,
|
|
25
|
+
entry.size,
|
|
26
|
+
entry.role,
|
|
27
|
+
entry.sourceStep ?? null,
|
|
28
|
+
entry.sourceRun ?? null,
|
|
29
|
+
entry.fileType ?? null,
|
|
30
|
+
entry.fileId ?? null,
|
|
31
|
+
now,
|
|
32
|
+
],
|
|
33
|
+
});
|
|
31
34
|
});
|
|
32
35
|
}
|
|
33
36
|
const COLS_PER_ROW = 10;
|
|
@@ -49,31 +52,32 @@ const ROWS_PER_STATEMENT = 1_000;
|
|
|
49
52
|
* idempotent per row (`ON CONFLICT ... DO UPDATE`), so a failure between
|
|
50
53
|
* chunks is healed by the caller's retry re-upserting the same manifest.
|
|
51
54
|
*/
|
|
52
|
-
export
|
|
55
|
+
export function upsertArtifacts(pool, entries) {
|
|
53
56
|
const now = new Date().toISOString();
|
|
54
|
-
|
|
55
|
-
|
|
56
|
-
|
|
57
|
-
|
|
58
|
-
|
|
59
|
-
|
|
60
|
-
|
|
61
|
-
|
|
62
|
-
|
|
63
|
-
|
|
64
|
-
e
|
|
65
|
-
|
|
66
|
-
|
|
67
|
-
|
|
68
|
-
|
|
69
|
-
|
|
70
|
-
|
|
71
|
-
|
|
72
|
-
|
|
73
|
-
|
|
74
|
-
|
|
75
|
-
|
|
76
|
-
|
|
57
|
+
return tryMutation("artifacts.upsertArtifacts", async () => {
|
|
58
|
+
for (let start = 0; start < entries.length; start += ROWS_PER_STATEMENT) {
|
|
59
|
+
const chunk = entries.slice(start, start + ROWS_PER_STATEMENT);
|
|
60
|
+
const placeholders = chunk
|
|
61
|
+
.map((_, i) => {
|
|
62
|
+
const base = i * COLS_PER_ROW;
|
|
63
|
+
return (`($${base + 1}, $${base + 2}, $${base + 3}, $${base + 4}, ` +
|
|
64
|
+
`$${base + 5}, $${base + 6}, $${base + 7}, $${base + 8}, $${base + 9}, $${base + 10})`);
|
|
65
|
+
})
|
|
66
|
+
.join(", ");
|
|
67
|
+
const values = chunk.flatMap((e) => [
|
|
68
|
+
e.resourceId,
|
|
69
|
+
e.path,
|
|
70
|
+
e.hash,
|
|
71
|
+
e.size,
|
|
72
|
+
e.role,
|
|
73
|
+
e.sourceStep ?? null,
|
|
74
|
+
e.sourceRun ?? null,
|
|
75
|
+
e.fileType ?? null,
|
|
76
|
+
e.fileId ?? null,
|
|
77
|
+
now,
|
|
78
|
+
]);
|
|
79
|
+
await pool.query({
|
|
80
|
+
text: `INSERT INTO cortex_artifacts
|
|
77
81
|
(analysis_id, path, hash, size, role, source_step, source_run, file_type, file_id, created_at)
|
|
78
82
|
VALUES ${placeholders}
|
|
79
83
|
ON CONFLICT (analysis_id, path) DO UPDATE SET
|
|
@@ -84,42 +88,47 @@ export async function upsertArtifacts(pool, entries) {
|
|
|
84
88
|
source_run = EXCLUDED.source_run,
|
|
85
89
|
file_type = COALESCE(EXCLUDED.file_type, cortex_artifacts.file_type),
|
|
86
90
|
file_id = COALESCE(EXCLUDED.file_id, cortex_artifacts.file_id)`,
|
|
87
|
-
|
|
88
|
-
|
|
89
|
-
|
|
91
|
+
values,
|
|
92
|
+
});
|
|
93
|
+
}
|
|
94
|
+
});
|
|
90
95
|
}
|
|
91
96
|
/**
|
|
92
97
|
* Query cortex_artifacts for input-role rows matching the given paths.
|
|
93
98
|
* Used at the registration boundary to resolve metadata for data-source inputs.
|
|
94
99
|
*/
|
|
95
|
-
export
|
|
96
|
-
|
|
97
|
-
|
|
98
|
-
|
|
99
|
-
|
|
100
|
+
export function queryInputArtifacts(pool, analysisId, paths) {
|
|
101
|
+
return tryQuery("artifacts.queryInputArtifacts", async () => {
|
|
102
|
+
if (paths.length === 0)
|
|
103
|
+
return [];
|
|
104
|
+
const result = await pool.query({
|
|
105
|
+
text: `SELECT path, hash, size, file_id
|
|
100
106
|
FROM cortex_artifacts
|
|
101
107
|
WHERE analysis_id = $1 AND role = 'input' AND path = ANY($2::text[])`,
|
|
102
|
-
|
|
108
|
+
values: [analysisId, paths],
|
|
109
|
+
});
|
|
110
|
+
return result.rows.map((r) => ({
|
|
111
|
+
path: r.path,
|
|
112
|
+
hash: r.hash,
|
|
113
|
+
size: Number(r.size),
|
|
114
|
+
fileId: r.file_id ?? null,
|
|
115
|
+
}));
|
|
103
116
|
});
|
|
104
|
-
return result.rows.map((r) => ({
|
|
105
|
-
path: r.path,
|
|
106
|
-
hash: r.hash,
|
|
107
|
-
size: Number(r.size),
|
|
108
|
-
fileId: r.file_id ?? null,
|
|
109
|
-
}));
|
|
110
117
|
}
|
|
111
|
-
export
|
|
112
|
-
|
|
113
|
-
|
|
118
|
+
export function queryUnsyncedStepArtifacts(pool, resourceId, runId, stepId) {
|
|
119
|
+
return tryQuery("artifacts.queryUnsyncedStepArtifacts", async () => {
|
|
120
|
+
const result = await pool.query({
|
|
121
|
+
text: `SELECT analysis_id, path, artifact_id, file_id, hash, size, role,
|
|
114
122
|
source_step, source_run, created_at, file_type
|
|
115
123
|
FROM cortex_artifacts
|
|
116
124
|
WHERE analysis_id = $1 AND source_run = $2 AND source_step = $3
|
|
117
125
|
AND artifact_id IS NOT NULL AND file_id IS NULL
|
|
118
126
|
AND role = 'step_output'
|
|
119
127
|
ORDER BY created_at`,
|
|
120
|
-
|
|
128
|
+
values: [resourceId, runId, stepId],
|
|
129
|
+
});
|
|
130
|
+
return result.rows.map(mapArtifactRow);
|
|
121
131
|
});
|
|
122
|
-
return result.rows.map(mapArtifactRow);
|
|
123
132
|
}
|
|
124
133
|
/**
|
|
125
134
|
* The artifacts a completed step registered, most consequential first. Ordered
|
|
@@ -127,11 +136,12 @@ export async function queryUnsyncedStepArtifacts(pool, resourceId, runId, stepId
|
|
|
127
136
|
* path, so a `limit` truncates the tail of the list rather than an arbitrary
|
|
128
137
|
* slice of it — and so the same call replays to the same rows.
|
|
129
138
|
*/
|
|
130
|
-
export
|
|
131
|
-
|
|
132
|
-
|
|
133
|
-
|
|
134
|
-
|
|
139
|
+
export function queryStepArtifactPaths(pool, resourceId, runId, stepId, limit) {
|
|
140
|
+
return tryQuery("artifacts.queryStepArtifactPaths", async () => {
|
|
141
|
+
if (limit <= 0)
|
|
142
|
+
return [];
|
|
143
|
+
const result = await pool.query({
|
|
144
|
+
text: `SELECT path, file_type
|
|
135
145
|
FROM cortex_artifacts
|
|
136
146
|
WHERE analysis_id = $1 AND source_run = $2 AND source_step = $3
|
|
137
147
|
AND role = 'step_output'
|
|
@@ -144,9 +154,10 @@ export async function queryStepArtifactPaths(pool, resourceId, runId, stepId, li
|
|
|
144
154
|
END,
|
|
145
155
|
path
|
|
146
156
|
LIMIT $4`,
|
|
147
|
-
|
|
157
|
+
values: [resourceId, runId, stepId, limit],
|
|
158
|
+
});
|
|
159
|
+
return result.rows.map((r) => ({ path: r.path, fileType: r.file_type ?? null }));
|
|
148
160
|
});
|
|
149
|
-
return result.rows.map((r) => ({ path: r.path, fileType: r.file_type ?? null }));
|
|
150
161
|
}
|
|
151
162
|
/**
|
|
152
163
|
* Every artifact of one analysis, ordered by path so the same call gives the same rows.
|
|
@@ -156,50 +167,58 @@ export async function queryStepArtifactPaths(pool, resourceId, runId, stepId, li
|
|
|
156
167
|
* query drops such a row, a later refusal says that the artifact never existed, and that reason is
|
|
157
168
|
* false.
|
|
158
169
|
*/
|
|
159
|
-
export
|
|
160
|
-
|
|
161
|
-
|
|
170
|
+
export function queryAnalysisArtifacts(pool, analysisId) {
|
|
171
|
+
return tryQuery("artifacts.queryAnalysisArtifacts", async () => {
|
|
172
|
+
const result = await pool.query({
|
|
173
|
+
text: `SELECT path, hash, file_type
|
|
162
174
|
FROM cortex_artifacts
|
|
163
175
|
WHERE analysis_id = $1
|
|
164
176
|
ORDER BY path`,
|
|
165
|
-
|
|
177
|
+
values: [analysisId],
|
|
178
|
+
});
|
|
179
|
+
return result.rows.map((r) => ({ path: r.path, hash: r.hash, fileType: r.file_type ?? null }));
|
|
166
180
|
});
|
|
167
|
-
return result.rows.map((r) => ({ path: r.path, hash: r.hash, fileType: r.file_type ?? null }));
|
|
168
181
|
}
|
|
169
182
|
/** Count step-output artifacts produced by a run — for the run-completed card. */
|
|
170
|
-
export
|
|
171
|
-
|
|
172
|
-
|
|
183
|
+
export function countArtifactsForRun(pool, analysisId, runId) {
|
|
184
|
+
return tryQuery("artifacts.countArtifactsForRun", async () => {
|
|
185
|
+
const result = await pool.query({
|
|
186
|
+
text: `SELECT COUNT(*)::text AS n
|
|
173
187
|
FROM cortex_artifacts
|
|
174
188
|
WHERE analysis_id = $1 AND source_run = $2 AND role = 'step_output'`,
|
|
175
|
-
|
|
189
|
+
values: [analysisId, runId],
|
|
190
|
+
});
|
|
191
|
+
return Number(result.rows[0]?.n ?? 0);
|
|
176
192
|
});
|
|
177
|
-
return Number(result.rows[0]?.n ?? 0);
|
|
178
193
|
}
|
|
179
|
-
export
|
|
180
|
-
|
|
181
|
-
|
|
194
|
+
export function updateArtifactId(pool, resourceId, path, artifactId, fileType) {
|
|
195
|
+
return tryMutation("artifacts.updateArtifactId", async () => {
|
|
196
|
+
await pool.query({
|
|
197
|
+
text: `UPDATE cortex_artifacts SET artifact_id = $1,
|
|
182
198
|
file_type = COALESCE($4, file_type)
|
|
183
199
|
WHERE analysis_id = $2 AND path = $3`,
|
|
184
|
-
|
|
200
|
+
values: [artifactId, resourceId, path, fileType ?? null],
|
|
201
|
+
});
|
|
185
202
|
});
|
|
186
203
|
}
|
|
187
|
-
export
|
|
188
|
-
|
|
189
|
-
|
|
190
|
-
|
|
191
|
-
|
|
192
|
-
|
|
193
|
-
|
|
194
|
-
|
|
195
|
-
|
|
196
|
-
|
|
197
|
-
|
|
198
|
-
|
|
204
|
+
export function updateFileIds(pool, pairs) {
|
|
205
|
+
return tryMutation("artifacts.updateFileIds", async () => {
|
|
206
|
+
if (pairs.length === 0)
|
|
207
|
+
return;
|
|
208
|
+
const values = [];
|
|
209
|
+
const tuples = [];
|
|
210
|
+
for (let i = 0; i < pairs.length; i++) {
|
|
211
|
+
const base = i * 2;
|
|
212
|
+
tuples.push(`($${base + 1}::text, $${base + 2}::text)`);
|
|
213
|
+
values.push(pairs[i].artifactId, pairs[i].fileId);
|
|
214
|
+
}
|
|
215
|
+
await pool.query({
|
|
216
|
+
text: `UPDATE cortex_artifacts AS a
|
|
199
217
|
SET file_id = c.file_id
|
|
200
218
|
FROM (VALUES ${tuples.join(", ")}) AS c(artifact_id, file_id)
|
|
201
219
|
WHERE a.artifact_id = c.artifact_id`,
|
|
202
|
-
|
|
220
|
+
values,
|
|
221
|
+
});
|
|
203
222
|
});
|
|
204
223
|
}
|
|
205
224
|
function mapArtifactRow(row) {
|
|
@@ -146,9 +146,9 @@ export declare function expireStaleDataProfile(pool: Querier, analysisId: string
|
|
|
146
146
|
* Reset a ledger row wedged at `running` with no workflow behind it back to
|
|
147
147
|
* `failed`, so the normal retry path can re-profile it.
|
|
148
148
|
*
|
|
149
|
-
* A start that
|
|
149
|
+
* A start that fails after the CAS already flipped the row to `running`
|
|
150
150
|
* compensates itself (`triggerDataProfile`/`runDataProfile` fail the ledger in
|
|
151
|
-
* their
|
|
151
|
+
* their dispatch). This covers the residual case that compensation cannot: a host
|
|
152
152
|
* that dies in the window between the CAS and the `DBOS.startWorkflow` insert
|
|
153
153
|
* leaves a `running` row with no workflow for recovery to resume — nothing
|
|
154
154
|
* would ever move it off `running`, and every later trigger reports
|
|
@@ -225,9 +225,9 @@ export function expireStaleDataProfile(pool, analysisId, timeoutMs) {
|
|
|
225
225
|
* Reset a ledger row wedged at `running` with no workflow behind it back to
|
|
226
226
|
* `failed`, so the normal retry path can re-profile it.
|
|
227
227
|
*
|
|
228
|
-
* A start that
|
|
228
|
+
* A start that fails after the CAS already flipped the row to `running`
|
|
229
229
|
* compensates itself (`triggerDataProfile`/`runDataProfile` fail the ledger in
|
|
230
|
-
* their
|
|
230
|
+
* their dispatch). This covers the residual case that compensation cannot: a host
|
|
231
231
|
* that dies in the window between the CAS and the `DBOS.startWorkflow` insert
|
|
232
232
|
* leaves a `running` row with no workflow for recovery to resume — nothing
|
|
233
233
|
* would ever move it off `running`, and every later trigger reports
|
|
@@ -17,7 +17,7 @@
|
|
|
17
17
|
* triggers dedup via the ledger CAS (only one `tryStart/tryRerun/tryRetry`
|
|
18
18
|
* UPDATE wins); each winning attempt starts a workflow under a per-attempt id.
|
|
19
19
|
*/
|
|
20
|
-
import {
|
|
20
|
+
import { ResultAsync } from "neverthrow";
|
|
21
21
|
import type { Pool } from "pg";
|
|
22
22
|
import { type AuthContext, type RunSession } from "../auth/types.js";
|
|
23
23
|
import type { EnvironmentStorePaths } from "../config/environment-stores.js";
|
|
@@ -26,6 +26,7 @@ import type { RunAuthorization, RunAuthorizer } from "../execution/run-authorize
|
|
|
26
26
|
import type { StagedInput } from "../execution/staged-input.js";
|
|
27
27
|
import type { SandboxAgentDeps } from "../agents/sandbox/shared.js";
|
|
28
28
|
import type { BioToolKeys } from "../tools/bio/keys.js";
|
|
29
|
+
import type { DbError } from "../lib/db-result.js";
|
|
29
30
|
import { type GateRefusal } from "../lib/hooks.js";
|
|
30
31
|
import type { Logger } from "../lib/logger.js";
|
|
31
32
|
import type { UsageRecorder } from "../billing/usage-recorder.js";
|
|
@@ -149,9 +150,25 @@ export declare function runDataProfileBody(input: DataProfileWorkflowInput, deps
|
|
|
149
150
|
* - `"already_running"`: a workflow already owns the row, and nothing new ran.
|
|
150
151
|
* - `"completed"`: the analysis has no input files, so the row is `completed` at once
|
|
151
152
|
* with no result. No workflow ran (see {@link completeEmptyDataProfile}).
|
|
152
|
-
* - `"failed"`: the trigger refused the call
|
|
153
|
+
* - `"failed"`: the trigger refused the call, and the row is unchanged.
|
|
153
154
|
*/
|
|
154
155
|
export type DataProfileTriggerResult = "started" | "restarted" | "already_running" | "completed" | "failed";
|
|
156
|
+
/**
|
|
157
|
+
* Why the dispatch of a claimed profile started no workflow. The dispatch writes `failed` to the claimed
|
|
158
|
+
* row before a caller reads the `err`:
|
|
159
|
+
* - `refused`: `RunAuthorizer.authorize` gave an `err`. `reason` and `suspend` are the values of the host.
|
|
160
|
+
* - `start_failed`: DBOS did not start the workflow. `cause` is the throw of DBOS.
|
|
161
|
+
*/
|
|
162
|
+
export type DataProfileStartError = {
|
|
163
|
+
readonly type: "refused";
|
|
164
|
+
readonly reason: string;
|
|
165
|
+
readonly suspend: boolean;
|
|
166
|
+
} | {
|
|
167
|
+
readonly type: "start_failed";
|
|
168
|
+
readonly cause: unknown;
|
|
169
|
+
};
|
|
170
|
+
/** A one-line, user-facing description of a `DataProfileStartError`. */
|
|
171
|
+
export declare function describeDataProfileStartError(e: DataProfileStartError): string;
|
|
155
172
|
/**
|
|
156
173
|
* Route-side deps for triggering the data-profile workflow: the ledger pool,
|
|
157
174
|
* the run authorizer, and the registered workflow callable. The body's
|
|
@@ -213,16 +230,17 @@ export declare function dataProfileWorkflowId(analysisId: string, nonce: string)
|
|
|
213
230
|
* `"completed"`. An empty manifest against a seed that names files is a divergence
|
|
214
231
|
* between the caller and the ledger, and the trigger refuses it.
|
|
215
232
|
*
|
|
216
|
-
* Returns what happened, so the caller can surface it (e.g. in the seed response).
|
|
233
|
+
* Returns what happened, so the caller can surface it (e.g. in the seed response). A
|
|
234
|
+
* failed ledger read or claim is the `err`. The dispatch settles its own row, thus a
|
|
235
|
+
* refusal or a failed start after a claim shows on the ledger only.
|
|
217
236
|
*/
|
|
218
|
-
export declare function triggerDataProfile(deps: DataProfileTriggerDeps, params: DataProfileTriggerParams):
|
|
237
|
+
export declare function triggerDataProfile(deps: DataProfileTriggerDeps, params: DataProfileTriggerParams): ResultAsync<DataProfileTriggerResult, DbError>;
|
|
219
238
|
/**
|
|
220
239
|
* Start the data-profile workflow for an already-claimed analysis (the retry
|
|
221
240
|
* route claims via `tryRetryDataProfile`, then calls this). The caller does not
|
|
222
|
-
*
|
|
223
|
-
*
|
|
224
|
-
*
|
|
225
|
-
*
|
|
226
|
-
* promise resolves: the row is where a caller reads the outcome.
|
|
241
|
+
* wait for the completion of the workflow. A refused authorization and a failed
|
|
242
|
+
* start each settle the row as `failed` before the `err` (see
|
|
243
|
+
* {@link dispatchClaimedProfile}), thus a caller that minted an authorization
|
|
244
|
+
* can revoke it.
|
|
227
245
|
*/
|
|
228
|
-
export declare function runDataProfile(deps: DataProfileTriggerDeps, params: DataProfileTriggerParams):
|
|
246
|
+
export declare function runDataProfile(deps: DataProfileTriggerDeps, params: DataProfileTriggerParams): ResultAsync<void, DataProfileStartError>;
|
|
@@ -19,7 +19,7 @@
|
|
|
19
19
|
*/
|
|
20
20
|
import { DBOS, Error as DBOSErrors } from "@dbos-inc/dbos-sdk";
|
|
21
21
|
import { randomUUID } from "node:crypto";
|
|
22
|
-
import { err, ok } from "neverthrow";
|
|
22
|
+
import { ResultAsync, err, ok } from "neverthrow";
|
|
23
23
|
import { forStep, forSubAgent } from "../auth/types.js";
|
|
24
24
|
import { createDataProfilerAgent } from "../agents/sandbox/data-profiler.js";
|
|
25
25
|
import { runToTerminal } from "../loop/run-to-terminal.js";
|
|
@@ -345,14 +345,14 @@ export async function runDataProfileBody(input, deps) {
|
|
|
345
345
|
return;
|
|
346
346
|
}
|
|
347
347
|
// 2. Register staged files in cortex_artifacts
|
|
348
|
-
await upsertArtifacts(deps.pool, stagedInputs.map((f) => ({
|
|
348
|
+
unwrapOrThrow(await upsertArtifacts(deps.pool, stagedInputs.map((f) => ({
|
|
349
349
|
resourceId: analysisId,
|
|
350
350
|
path: inputArtifactPath(f),
|
|
351
351
|
hash: f.hash,
|
|
352
352
|
size: f.size,
|
|
353
353
|
role: "input",
|
|
354
354
|
fileId: f.fileId,
|
|
355
|
-
})));
|
|
355
|
+
}))));
|
|
356
356
|
const executionId = generateExecutionId(DATA_PROFILE_AGENT_ID);
|
|
357
357
|
const workflowId = DBOS.workflowID ?? `${DATA_PROFILE_RUN_LITERAL}:${executionId}`;
|
|
358
358
|
const childSession = forSubAgent(runSession, DATA_PROFILE_AGENT_ID);
|
|
@@ -636,6 +636,15 @@ function settleTerminalWrite(logger, analysisId, write, status) {
|
|
|
636
636
|
}
|
|
637
637
|
recordRunCompleted({ workflow: "data_profile", status, durationMs: elapsedSinceIso(write.startedAt) });
|
|
638
638
|
}
|
|
639
|
+
/** A one-line, user-facing description of a `DataProfileStartError`. */
|
|
640
|
+
export function describeDataProfileStartError(e) {
|
|
641
|
+
switch (e.type) {
|
|
642
|
+
case "refused":
|
|
643
|
+
return `the profile run was not authorized: ${e.reason}`;
|
|
644
|
+
case "start_failed":
|
|
645
|
+
return `the profile workflow did not start: ${profileFailureReason(e.cause)}`;
|
|
646
|
+
}
|
|
647
|
+
}
|
|
639
648
|
/**
|
|
640
649
|
* Resolve the `RunSession` the workflow runs under. Whether the authorization
|
|
641
650
|
* is freshly minted (Cortex-owned) or reused (caller-owned) is decided by the
|
|
@@ -667,18 +676,14 @@ export function dataProfileWorkflowId(analysisId, nonce) {
|
|
|
667
676
|
return `dataprofile:${analysisId}:${nonce}`;
|
|
668
677
|
}
|
|
669
678
|
/**
|
|
670
|
-
*
|
|
671
|
-
*
|
|
672
|
-
*
|
|
673
|
-
*
|
|
674
|
-
*
|
|
675
|
-
* not awaited.
|
|
679
|
+
* Start the data-profile workflow under the authorization and a per-attempt id
|
|
680
|
+
* `dataprofile:{analysisId}:{nonce}` — concurrent triggers are already
|
|
681
|
+
* serialized by the ledger CAS. The caller has already staged the inputs and
|
|
682
|
+
* supplied the manifest in `params.stagedInputs`; this forwards it into the
|
|
683
|
+
* workflow input. The handle result is not awaited.
|
|
676
684
|
*/
|
|
677
|
-
async function startDataProfileWorkflow(deps, params) {
|
|
678
|
-
const
|
|
679
|
-
if (authorized.isErr())
|
|
680
|
-
return err(authorized.error);
|
|
681
|
-
const { runSession, ownsMandate } = authorized.value; // oss-core-managed-ok
|
|
685
|
+
async function startDataProfileWorkflow(deps, params, authorization) {
|
|
686
|
+
const { runSession, ownsMandate } = authorization; // oss-core-managed-ok
|
|
682
687
|
const attemptNonce = randomUUID();
|
|
683
688
|
await DBOS.startWorkflow(deps.workflow, {
|
|
684
689
|
workflowID: dataProfileWorkflowId(params.analysisId, attemptNonce),
|
|
@@ -688,20 +693,29 @@ async function startDataProfileWorkflow(deps, params) {
|
|
|
688
693
|
ownsMandate, // oss-core-managed-ok
|
|
689
694
|
stagedInputs: params.stagedInputs,
|
|
690
695
|
});
|
|
691
|
-
return ok(undefined);
|
|
692
696
|
}
|
|
693
697
|
/**
|
|
694
|
-
*
|
|
695
|
-
*
|
|
696
|
-
*
|
|
698
|
+
* Authorize and start a claimed profile. The claim already flipped the row to
|
|
699
|
+
* `running`, thus a refused authorization and a failed start each settle the
|
|
700
|
+
* row as `failed` before the `err`, and the row never wedges at `running`. A
|
|
701
|
+
* rejected `authorize` promise is a defect of the host: the row is settled the
|
|
702
|
+
* same way, and the rejection passes through.
|
|
697
703
|
*/
|
|
698
704
|
async function dispatchClaimedProfile(deps, params, phase) {
|
|
699
|
-
const
|
|
700
|
-
await compensateStartFailure(deps, params.analysisId, phase,
|
|
701
|
-
|
|
705
|
+
const authorized = await Promise.resolve(authorizeDataProfile(deps, params)).catch(async (defect) => {
|
|
706
|
+
await compensateStartFailure(deps, params.analysisId, phase, defect);
|
|
707
|
+
throw defect;
|
|
702
708
|
});
|
|
703
|
-
if (
|
|
704
|
-
await refuseClaimedProfile(deps, params.analysisId, phase,
|
|
709
|
+
if (authorized.isErr()) {
|
|
710
|
+
await refuseClaimedProfile(deps, params.analysisId, phase, authorized.error);
|
|
711
|
+
return err({ type: "refused", reason: authorized.error.reason, suspend: authorized.error.kind === "suspended" });
|
|
712
|
+
}
|
|
713
|
+
const started = await ResultAsync.fromPromise(startDataProfileWorkflow(deps, params, authorized.value), (cause) => cause);
|
|
714
|
+
if (started.isErr()) {
|
|
715
|
+
await compensateStartFailure(deps, params.analysisId, phase, started.error);
|
|
716
|
+
return err({ type: "start_failed", cause: started.error });
|
|
717
|
+
}
|
|
718
|
+
return ok(undefined);
|
|
705
719
|
}
|
|
706
720
|
/**
|
|
707
721
|
* Attempt to claim and run data profiling for an analysis, then start the workflow
|
|
@@ -720,70 +734,85 @@ async function dispatchClaimedProfile(deps, params, phase) {
|
|
|
720
734
|
* `"completed"`. An empty manifest against a seed that names files is a divergence
|
|
721
735
|
* between the caller and the ledger, and the trigger refuses it.
|
|
722
736
|
*
|
|
723
|
-
* Returns what happened, so the caller can surface it (e.g. in the seed response).
|
|
737
|
+
* Returns what happened, so the caller can surface it (e.g. in the seed response). A
|
|
738
|
+
* failed ledger read or claim is the `err`. The dispatch settles its own row, thus a
|
|
739
|
+
* refusal or a failed start after a claim shows on the ledger only.
|
|
724
740
|
*/
|
|
725
|
-
export
|
|
741
|
+
export function triggerDataProfile(deps, params) {
|
|
742
|
+
return new ResultAsync(claimAndDispatchProfile(deps, params));
|
|
743
|
+
}
|
|
744
|
+
async function claimAndDispatchProfile(deps, params) {
|
|
726
745
|
const logger = (deps.logger ?? createNoopLogger()).named("data-profile").with({ analysisId: params.analysisId });
|
|
727
746
|
const { analysisId } = params;
|
|
728
|
-
|
|
729
|
-
|
|
730
|
-
|
|
731
|
-
|
|
732
|
-
|
|
733
|
-
|
|
734
|
-
|
|
735
|
-
|
|
736
|
-
|
|
737
|
-
|
|
738
|
-
|
|
739
|
-
|
|
740
|
-
|
|
741
|
-
|
|
742
|
-
|
|
743
|
-
|
|
744
|
-
|
|
745
|
-
}
|
|
746
|
-
|
|
747
|
-
// `completed` at once, with no workflow. The CAS carries the same seed
|
|
748
|
-
// predicate as this pre-read, so a seed upsert that lands in between cannot be
|
|
749
|
-
// hidden behind a finished profile: the stamp refuses, and the status read
|
|
750
|
-
// below names the cause.
|
|
751
|
-
if (unwrapOrThrow(await completeEmptyDataProfile(deps.pool, analysisId)))
|
|
752
|
-
return "completed";
|
|
753
|
-
const current = unwrapOrThrow(await loadDataProfileStatus(deps.pool, analysisId));
|
|
754
|
-
if (current?.status === "running")
|
|
755
|
-
return "already_running";
|
|
756
|
-
logger.error("trigger rejected: empty-set completion refused (no analysis row, or the seed now names files)");
|
|
757
|
-
return "failed";
|
|
758
|
-
}
|
|
759
|
-
// A non-empty manifest against a seed that names no file: the caller skipped the
|
|
760
|
-
// seed, or the analysis row is missing. Name the cause before any claim.
|
|
761
|
-
if (seeded === null || seeded.length === 0) {
|
|
762
|
-
logger.error("trigger rejected: no seeded input set (missing analysis row or caller skipped seeding)");
|
|
763
|
-
return "failed";
|
|
764
|
-
}
|
|
765
|
-
const started = unwrapOrThrow(await tryStartDataProfile(deps.pool, analysisId));
|
|
766
|
-
if (started) {
|
|
767
|
-
void dispatchClaimedProfile(deps, params, "Run");
|
|
768
|
-
return "started";
|
|
747
|
+
// ADVISORY pre-check, not the enforcement — the claim CAS carries the same seed
|
|
748
|
+
// conjunct (see `SEEDED` in state/data-profile.ts) and is what actually makes a
|
|
749
|
+
// seedless `running` row impossible. This read exists only to name the cause: on
|
|
750
|
+
// the ordinary non-racing path it turns an opaque "no claim matched" into a
|
|
751
|
+
// precise reason. An empty array is not a seed — it names zero files. Reads via
|
|
752
|
+
// `loadSeedInputFileIds`, which returns null for BOTH a missing analysis row and
|
|
753
|
+
// a NULL-seed row (`loadDataProfileStatus` would hide the seed of a cleared row).
|
|
754
|
+
const seedRead = await loadSeedInputFileIds(deps.pool, analysisId);
|
|
755
|
+
if (seedRead.isErr())
|
|
756
|
+
return err(seedRead.error);
|
|
757
|
+
const seeded = seedRead.value;
|
|
758
|
+
if (params.stagedInputs.length === 0) {
|
|
759
|
+
// The trigger validates the manifest it is about to dispatch against the ledger
|
|
760
|
+
// seed. A seeded row profiled against an EMPTY manifest would claim `running` and
|
|
761
|
+
// then hit the body's empty-manifest path, and complete with a NULL result while
|
|
762
|
+
// the seed still names files. Refuse the divergence before any claim.
|
|
763
|
+
if (seeded !== null && seeded.length > 0) {
|
|
764
|
+
logger.error("trigger rejected: empty manifest dispatched against a non-empty seed", { seededCount: seeded.length });
|
|
765
|
+
return ok("failed");
|
|
769
766
|
}
|
|
770
|
-
|
|
771
|
-
|
|
772
|
-
|
|
773
|
-
|
|
774
|
-
|
|
775
|
-
const
|
|
776
|
-
if (
|
|
777
|
-
return
|
|
778
|
-
|
|
767
|
+
// The analysis has no input files, so there is nothing to profile. The row is
|
|
768
|
+
// `completed` at once, with no workflow. The CAS carries the same seed
|
|
769
|
+
// predicate as this pre-read, so a seed upsert that lands in between cannot be
|
|
770
|
+
// hidden behind a finished profile: the stamp refuses, and the status read
|
|
771
|
+
// below names the cause.
|
|
772
|
+
const completed = await completeEmptyDataProfile(deps.pool, analysisId);
|
|
773
|
+
if (completed.isErr())
|
|
774
|
+
return err(completed.error);
|
|
775
|
+
if (completed.value)
|
|
776
|
+
return ok("completed");
|
|
777
|
+
const current = await loadDataProfileStatus(deps.pool, analysisId);
|
|
778
|
+
if (current.isErr())
|
|
779
|
+
return err(current.error);
|
|
780
|
+
if (current.value?.status === "running")
|
|
781
|
+
return ok("already_running");
|
|
782
|
+
logger.error("trigger rejected: empty-set completion refused (no analysis row, or the seed now names files)");
|
|
783
|
+
return ok("failed");
|
|
779
784
|
}
|
|
780
|
-
|
|
781
|
-
|
|
782
|
-
|
|
785
|
+
// A non-empty manifest against a seed that names no file: the caller skipped the
|
|
786
|
+
// seed, or the analysis row is missing. Name the cause before any claim.
|
|
787
|
+
if (seeded === null || seeded.length === 0) {
|
|
788
|
+
logger.error("trigger rejected: no seeded input set (missing analysis row or caller skipped seeding)");
|
|
789
|
+
return ok("failed");
|
|
790
|
+
}
|
|
791
|
+
const started = await tryStartDataProfile(deps.pool, analysisId);
|
|
792
|
+
if (started.isErr())
|
|
793
|
+
return err(started.error);
|
|
794
|
+
if (started.value) {
|
|
795
|
+
dispatchInBackground(deps, params, "Run");
|
|
796
|
+
return ok("started");
|
|
797
|
+
}
|
|
798
|
+
const restarted = await tryRerunDataProfile(deps.pool, analysisId);
|
|
799
|
+
if (restarted.isErr())
|
|
800
|
+
return err(restarted.error);
|
|
801
|
+
if (restarted.value) {
|
|
802
|
+
dispatchInBackground(deps, params, "Re-run");
|
|
803
|
+
return ok("restarted");
|
|
783
804
|
}
|
|
805
|
+
const status = await loadDataProfileStatus(deps.pool, analysisId);
|
|
806
|
+
if (status.isErr())
|
|
807
|
+
return err(status.error);
|
|
808
|
+
return ok(status.value?.status === "running" ? "already_running" : "failed");
|
|
809
|
+
}
|
|
810
|
+
/** The dispatch settles the row on each arm and logs a defect, thus the trigger keeps no outcome of it. */
|
|
811
|
+
function dispatchInBackground(deps, params, phase) {
|
|
812
|
+
void dispatchClaimedProfile(deps, params, phase).catch(() => undefined);
|
|
784
813
|
}
|
|
785
814
|
/**
|
|
786
|
-
* Compensate a
|
|
815
|
+
* Compensate a start that failed after the claim. The ledger CAS in
|
|
787
816
|
* `tryStart`/`tryRerun`/`tryRetry` already flipped the row to `running`; if the
|
|
788
817
|
* dispatch never landed a workflow, DBOS has nothing to recover, so without
|
|
789
818
|
* this the row would sit at `running` forever and every later trigger would
|
|
@@ -827,23 +856,13 @@ async function failClaimedProfile(deps, analysisId, phase, reason) {
|
|
|
827
856
|
/**
|
|
828
857
|
* Start the data-profile workflow for an already-claimed analysis (the retry
|
|
829
858
|
* route claims via `tryRetryDataProfile`, then calls this). The caller does not
|
|
830
|
-
*
|
|
831
|
-
*
|
|
832
|
-
*
|
|
833
|
-
*
|
|
834
|
-
* promise resolves: the row is where a caller reads the outcome.
|
|
859
|
+
* wait for the completion of the workflow. A refused authorization and a failed
|
|
860
|
+
* start each settle the row as `failed` before the `err` (see
|
|
861
|
+
* {@link dispatchClaimedProfile}), thus a caller that minted an authorization
|
|
862
|
+
* can revoke it.
|
|
835
863
|
*/
|
|
836
|
-
export
|
|
837
|
-
|
|
838
|
-
try {
|
|
839
|
-
started = await startDataProfileWorkflow(deps, params);
|
|
840
|
-
}
|
|
841
|
-
catch (cause) {
|
|
842
|
-
await compensateStartFailure(deps, params.analysisId, "Retry", cause);
|
|
843
|
-
throw cause;
|
|
844
|
-
}
|
|
845
|
-
if (started.isErr())
|
|
846
|
-
await refuseClaimedProfile(deps, params.analysisId, "Retry", started.error);
|
|
864
|
+
export function runDataProfile(deps, params) {
|
|
865
|
+
return new ResultAsync(dispatchClaimedProfile(deps, params, "Retry"));
|
|
847
866
|
}
|
|
848
867
|
/**
|
|
849
868
|
* Concise, UI-safe failure reason. `data_profile_error` is surfaced verbatim
|
|
@@ -69,7 +69,7 @@
|
|
|
69
69
|
* via `inspectRun`)
|
|
70
70
|
*/
|
|
71
71
|
import { DBOS, Error as DBOSErrors } from "@dbos-inc/dbos-sdk";
|
|
72
|
-
import {
|
|
72
|
+
import { err, ok } from "neverthrow";
|
|
73
73
|
import { mkdir } from "node:fs/promises";
|
|
74
74
|
import { join } from "node:path";
|
|
75
75
|
import { forStep } from "../auth/types.js";
|
|
@@ -286,7 +286,7 @@ async function collectUpstreamHandoffs(args) {
|
|
|
286
286
|
const handoffs = [];
|
|
287
287
|
for (const summary of summaries) {
|
|
288
288
|
const outputDir = stepSubdir(runStepDir(runId, summary.stepId), "output");
|
|
289
|
-
const artifacts = await queryStepArtifactPaths(pool, analysisId, runId, summary.stepId, MAX_UPSTREAM_ARTIFACTS);
|
|
289
|
+
const artifacts = unwrapOrThrow(await queryStepArtifactPaths(pool, analysisId, runId, summary.stepId, MAX_UPSTREAM_ARTIFACTS));
|
|
290
290
|
handoffs.push({
|
|
291
291
|
stepId: summary.stepId,
|
|
292
292
|
agentId: summary.agentId,
|
|
@@ -1124,7 +1124,9 @@ async function collectAndComplete(args) {
|
|
|
1124
1124
|
// beat the CLI's poll-and-shutdown). These branches only fan out the UI stream part, whose
|
|
1125
1125
|
// completed/failed shapes genuinely differ — the provenance record does not.
|
|
1126
1126
|
if (status === "completed" || status === "partial") {
|
|
1127
|
-
const artifactCount = await DBOS.runStep(async () => await
|
|
1127
|
+
const artifactCount = await DBOS.runStep(async () => await countArtifactsForRun(deps.pool, input.analysisId, runId).unwrapOr(0), {
|
|
1128
|
+
name: "count-run-artifacts",
|
|
1129
|
+
});
|
|
1128
1130
|
await emitStreamPart({
|
|
1129
1131
|
type: "data-run-completed",
|
|
1130
1132
|
runId,
|
package/package.json
CHANGED