@inflexa-ai/harness 0.35.2 → 0.35.4
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/citations/types.d.ts +2 -2
- package/dist/config/environment-stores.d.ts +10 -0
- package/dist/contracts/schemas/chat-parts.d.ts +11 -11
- package/dist/execution/run-canceler.js +20 -4
- package/dist/index.d.ts +3 -1
- package/dist/index.js +5 -0
- package/dist/lib/metrics.d.ts +63 -5
- package/dist/lib/metrics.js +104 -5
- package/dist/lib/otel-spans.d.ts +77 -0
- package/dist/lib/otel-spans.js +123 -0
- package/dist/lib/otel.d.ts +12 -3
- package/dist/lib/otel.js +15 -4
- package/dist/loop/run-agent.js +13 -2
- package/dist/prompts/planner.js +6 -0
- package/dist/prompts/sandbox-standards.d.ts +1 -1
- package/dist/prompts/sandbox-standards.js +8 -3
- package/dist/runtime/dbos.d.ts +8 -7
- package/dist/sandbox/await-exec.js +16 -9
- package/dist/sandbox/notification-sweep.js +5 -2
- package/dist/sandbox/package-identity.d.ts +208 -0
- package/dist/sandbox/package-identity.js +244 -0
- package/dist/sandbox/reaper.js +5 -2
- package/dist/sandbox/submit-exec.js +6 -3
- package/dist/sandbox/types.d.ts +20 -25
- package/dist/sandbox/watchdog.js +8 -3
- package/dist/schemas/plan-schemas.d.ts +3 -3
- package/dist/schemas/validate-plan.js +32 -15
- package/dist/schemas/workflow-state.js +3 -1
- package/dist/state/data-profile.d.ts +20 -6
- package/dist/state/data-profile.js +16 -8
- package/dist/state/index.d.ts +1 -1
- package/dist/state/schema.d.ts +2 -2
- package/dist/tasks/data-profile.js +26 -12
- package/dist/tools/execute-analysis.js +37 -20
- package/dist/tools/research/inspect-run.d.ts +1 -1
- package/dist/tools/sandbox/link-packages.d.ts +6 -6
- package/dist/tools/sandbox/link-packages.js +45 -14
- package/dist/tools/sandbox/list-available-packages.d.ts +23 -1
- package/dist/tools/sandbox/list-available-packages.js +185 -43
- package/dist/workflows/execute-analysis.d.ts +4 -1
- package/dist/workflows/execute-analysis.js +40 -4
- package/dist/workflows/execute-target-assessment.js +68 -31
- package/dist/workflows/sandbox-step.js +7 -1
- package/package.json +1 -1
|
@@ -48,9 +48,9 @@ export declare const CitationVerdictSchema: z.ZodEnum<{
|
|
|
48
48
|
}>;
|
|
49
49
|
export type CitationVerdict = z.infer<typeof CitationVerdictSchema>;
|
|
50
50
|
export declare const CitationCoverageSchema: z.ZodEnum<{
|
|
51
|
+
partial: "partial";
|
|
51
52
|
none: "none";
|
|
52
53
|
complete: "complete";
|
|
53
|
-
partial: "partial";
|
|
54
54
|
}>;
|
|
55
55
|
export type CitationCoverage = z.infer<typeof CitationCoverageSchema>;
|
|
56
56
|
export declare const ComparisonStatusSchema: z.ZodEnum<{
|
|
@@ -496,9 +496,9 @@ export declare const CitationResolutionResultSchema: z.ZodObject<{
|
|
|
496
496
|
inconclusive: "inconclusive";
|
|
497
497
|
}>;
|
|
498
498
|
coverage: z.ZodEnum<{
|
|
499
|
+
partial: "partial";
|
|
499
500
|
none: "none";
|
|
500
501
|
complete: "complete";
|
|
501
|
-
partial: "partial";
|
|
502
502
|
}>;
|
|
503
503
|
}, z.core.$strip>;
|
|
504
504
|
export type CitationResolutionResult = z.infer<typeof CitationResolutionResultSchema>;
|
|
@@ -8,6 +8,7 @@
|
|
|
8
8
|
* "the profiler", "the planner") while the contract stayed identical. Extending
|
|
9
9
|
* this interface keeps one description of a field that has one meaning.
|
|
10
10
|
*/
|
|
11
|
+
import type { Track } from "../sandbox/package-identity.js";
|
|
11
12
|
/**
|
|
12
13
|
* Host paths of the environment's two read-only stores.
|
|
13
14
|
*
|
|
@@ -94,5 +95,14 @@ export interface PoolInventoryPackage {
|
|
|
94
95
|
/** One language-track section of an inventory, as `list_available_packages` renders it. */
|
|
95
96
|
export interface PoolInventorySection {
|
|
96
97
|
readonly title: string;
|
|
98
|
+
/**
|
|
99
|
+
* The ecosystem that holds the packages of this section. It carries the
|
|
100
|
+
* track as DATA, thus no reader derives one from the title: a title is
|
|
101
|
+
* display text, and a heading that changes wording would silently change
|
|
102
|
+
* which packages a `language` filter answers. A section of system tools or
|
|
103
|
+
* of node packages carries none, because neither is a package track of the
|
|
104
|
+
* pool.
|
|
105
|
+
*/
|
|
106
|
+
readonly track?: Track;
|
|
97
107
|
readonly packages: readonly PoolInventoryPackage[];
|
|
98
108
|
}
|
|
@@ -104,8 +104,8 @@ export declare const FileReferencePartSchema: z.ZodObject<{
|
|
|
104
104
|
}, z.core.$strip>>;
|
|
105
105
|
}, z.core.$strip>;
|
|
106
106
|
export declare const AskStatusSchema: z.ZodEnum<{
|
|
107
|
-
aborted: "aborted";
|
|
108
107
|
pending: "pending";
|
|
108
|
+
aborted: "aborted";
|
|
109
109
|
resolved: "resolved";
|
|
110
110
|
rejected: "rejected";
|
|
111
111
|
expired: "expired";
|
|
@@ -117,8 +117,8 @@ export declare const AskPartSchema: z.ZodObject<{
|
|
|
117
117
|
command: z.ZodString;
|
|
118
118
|
detail: z.ZodOptional<z.ZodString>;
|
|
119
119
|
status: z.ZodEnum<{
|
|
120
|
-
aborted: "aborted";
|
|
121
120
|
pending: "pending";
|
|
121
|
+
aborted: "aborted";
|
|
122
122
|
resolved: "resolved";
|
|
123
123
|
rejected: "rejected";
|
|
124
124
|
expired: "expired";
|
|
@@ -182,8 +182,8 @@ export declare const DagStatePartSchema: z.ZodObject<{
|
|
|
182
182
|
}, z.core.$strict>>;
|
|
183
183
|
}, z.core.$strict>;
|
|
184
184
|
export declare const StepPhaseSchema: z.ZodEnum<{
|
|
185
|
-
complete: "complete";
|
|
186
185
|
failed: "failed";
|
|
186
|
+
complete: "complete";
|
|
187
187
|
warning: "warning";
|
|
188
188
|
"sandbox-init": "sandbox-init";
|
|
189
189
|
executing: "executing";
|
|
@@ -199,8 +199,8 @@ export declare const StepActivityPartSchema: z.ZodObject<{
|
|
|
199
199
|
runId: z.ZodString;
|
|
200
200
|
stepId: z.ZodString;
|
|
201
201
|
phase: z.ZodEnum<{
|
|
202
|
-
complete: "complete";
|
|
203
202
|
failed: "failed";
|
|
203
|
+
complete: "complete";
|
|
204
204
|
warning: "warning";
|
|
205
205
|
"sandbox-init": "sandbox-init";
|
|
206
206
|
executing: "executing";
|
|
@@ -314,8 +314,8 @@ export declare const RunCompletedPartSchema: z.ZodObject<{
|
|
|
314
314
|
type: z.ZodLiteral<"data-run-completed">;
|
|
315
315
|
runId: z.ZodString;
|
|
316
316
|
status: z.ZodEnum<{
|
|
317
|
-
partial: "partial";
|
|
318
317
|
completed: "completed";
|
|
318
|
+
partial: "partial";
|
|
319
319
|
}>;
|
|
320
320
|
completedSteps: z.ZodNumber;
|
|
321
321
|
totalSteps: z.ZodNumber;
|
|
@@ -403,9 +403,9 @@ export declare const RunSynthesisPartSchema: z.ZodObject<{
|
|
|
403
403
|
}, z.core.$strip>>;
|
|
404
404
|
}, z.core.$strip>;
|
|
405
405
|
export declare const SynthesisPhaseSchema: z.ZodEnum<{
|
|
406
|
-
complete: "complete";
|
|
407
406
|
failed: "failed";
|
|
408
407
|
skipped: "skipped";
|
|
408
|
+
complete: "complete";
|
|
409
409
|
indexing: "indexing";
|
|
410
410
|
persisting: "persisting";
|
|
411
411
|
starting: "starting";
|
|
@@ -419,9 +419,9 @@ export declare const SynthesisProgressPartSchema: z.ZodObject<{
|
|
|
419
419
|
id: z.ZodString;
|
|
420
420
|
runId: z.ZodString;
|
|
421
421
|
phase: z.ZodEnum<{
|
|
422
|
-
complete: "complete";
|
|
423
422
|
failed: "failed";
|
|
424
423
|
skipped: "skipped";
|
|
424
|
+
complete: "complete";
|
|
425
425
|
indexing: "indexing";
|
|
426
426
|
persisting: "persisting";
|
|
427
427
|
starting: "starting";
|
|
@@ -520,8 +520,8 @@ export declare const CortexChatPartSchema: z.ZodDiscriminatedUnion<[z.ZodObject<
|
|
|
520
520
|
command: z.ZodString;
|
|
521
521
|
detail: z.ZodOptional<z.ZodString>;
|
|
522
522
|
status: z.ZodEnum<{
|
|
523
|
-
aborted: "aborted";
|
|
524
523
|
pending: "pending";
|
|
524
|
+
aborted: "aborted";
|
|
525
525
|
resolved: "resolved";
|
|
526
526
|
rejected: "rejected";
|
|
527
527
|
expired: "expired";
|
|
@@ -560,8 +560,8 @@ export declare const CortexChatPartSchema: z.ZodDiscriminatedUnion<[z.ZodObject<
|
|
|
560
560
|
runId: z.ZodString;
|
|
561
561
|
stepId: z.ZodString;
|
|
562
562
|
phase: z.ZodEnum<{
|
|
563
|
-
complete: "complete";
|
|
564
563
|
failed: "failed";
|
|
564
|
+
complete: "complete";
|
|
565
565
|
warning: "warning";
|
|
566
566
|
"sandbox-init": "sandbox-init";
|
|
567
567
|
executing: "executing";
|
|
@@ -679,9 +679,9 @@ export declare const CortexChatPartSchema: z.ZodDiscriminatedUnion<[z.ZodObject<
|
|
|
679
679
|
id: z.ZodString;
|
|
680
680
|
runId: z.ZodString;
|
|
681
681
|
phase: z.ZodEnum<{
|
|
682
|
-
complete: "complete";
|
|
683
682
|
failed: "failed";
|
|
684
683
|
skipped: "skipped";
|
|
684
|
+
complete: "complete";
|
|
685
685
|
indexing: "indexing";
|
|
686
686
|
persisting: "persisting";
|
|
687
687
|
starting: "starting";
|
|
@@ -698,8 +698,8 @@ export declare const CortexChatPartSchema: z.ZodDiscriminatedUnion<[z.ZodObject<
|
|
|
698
698
|
type: z.ZodLiteral<"data-run-completed">;
|
|
699
699
|
runId: z.ZodString;
|
|
700
700
|
status: z.ZodEnum<{
|
|
701
|
-
partial: "partial";
|
|
702
701
|
completed: "completed";
|
|
702
|
+
partial: "partial";
|
|
703
703
|
}>;
|
|
704
704
|
completedSteps: z.ZodNumber;
|
|
705
705
|
totalSteps: z.ZodNumber;
|
|
@@ -18,6 +18,7 @@
|
|
|
18
18
|
* terminal frame read-side.
|
|
19
19
|
*/
|
|
20
20
|
import { createNoopLogger } from "../lib/console-logger.js";
|
|
21
|
+
import { elapsedSinceIso, recordRunCompleted, recordStepCompleted } from "../lib/metrics.js";
|
|
21
22
|
import { unwrapOrThrow } from "../lib/result.js";
|
|
22
23
|
import { markRunCanceledIfActive, queryRun } from "../state/runs.js";
|
|
23
24
|
import { queryStepsByRun, sweepPendingStepExecutions } from "../state/step-executions.js";
|
|
@@ -49,8 +50,9 @@ export function createRunCanceler(deps) {
|
|
|
49
50
|
})();
|
|
50
51
|
await engineCancel(workflowIds);
|
|
51
52
|
};
|
|
52
|
-
/**
|
|
53
|
-
const
|
|
53
|
+
/** Steps whose child started (a persisted child workflow id) and has not written its terminal row. */
|
|
54
|
+
const incompleteSteps = async (runId) => unwrapOrThrow(await queryStepsByRun(pool, runId)).filter((step) => step.childWorkflowId !== null && step.completedAt === null);
|
|
55
|
+
const childIds = (steps) => steps.flatMap((step) => (step.childWorkflowId !== null ? [step.childWorkflowId] : []));
|
|
54
56
|
return {
|
|
55
57
|
async cancel(runId, session) {
|
|
56
58
|
const row = unwrapOrThrow(await queryRun(pool, runId));
|
|
@@ -71,10 +73,14 @@ export function createRunCanceler(deps) {
|
|
|
71
73
|
// mark-running step has not yet committed its child_workflow_id. The
|
|
72
74
|
// persisted ids and the post-cancel re-query are belt-and-braces over
|
|
73
75
|
// both ledgers' lag.
|
|
74
|
-
const knownChildren = await
|
|
76
|
+
const knownChildren = childIds(await incompleteSteps(runId));
|
|
75
77
|
await cancelWorkflows([workflowId, ...knownChildren]);
|
|
78
|
+
// The steps the cancel cut: read after the cancel landed, so a child that
|
|
79
|
+
// wrote its terminal row in the meantime is not among them.
|
|
80
|
+
let cutSteps;
|
|
76
81
|
try {
|
|
77
|
-
|
|
82
|
+
cutSteps = await incompleteSteps(runId);
|
|
83
|
+
const late = childIds(cutSteps).filter((id) => !knownChildren.includes(id));
|
|
78
84
|
if (late.length > 0)
|
|
79
85
|
await cancelWorkflows(late);
|
|
80
86
|
}
|
|
@@ -89,6 +95,16 @@ export function createRunCanceler(deps) {
|
|
|
89
95
|
catch (err) {
|
|
90
96
|
logger.error("markRunCanceledIfActive failed", { runId, ...logger.errorFields(err) });
|
|
91
97
|
}
|
|
98
|
+
// The workflow's own terminal step never runs after a cancel, so this
|
|
99
|
+
// conditional write is the one terminal write of an external cancel, and
|
|
100
|
+
// the outcome metrics ride behind it: a refused write means the run
|
|
101
|
+
// settled on its own and its own terminal step recorded them. A cut
|
|
102
|
+
// child never reached its terminal row, so its step records here too.
|
|
103
|
+
if (transitioned) {
|
|
104
|
+
recordRunCompleted({ workflow: "analysis", status: "canceled", durationMs: elapsedSinceIso(row.startedAt) });
|
|
105
|
+
for (const step of cutSteps ?? [])
|
|
106
|
+
recordStepCompleted({ agentId: step.agentId, status: "canceled" });
|
|
107
|
+
}
|
|
92
108
|
let steps = false;
|
|
93
109
|
try {
|
|
94
110
|
unwrapOrThrow(await sweepPendingStepExecutions(pool, runId));
|
package/dist/index.d.ts
CHANGED
|
@@ -136,7 +136,9 @@ export type { CreateSandboxClientConfig, SandboxBackendConfig } from "./sandbox/
|
|
|
136
136
|
export type { SandboxClient } from "./sandbox/client.js";
|
|
137
137
|
export { describeSandboxError, SandboxFailure } from "./sandbox/sandbox-error.js";
|
|
138
138
|
export type { SandboxError } from "./sandbox/sandbox-error.js";
|
|
139
|
-
export type { ExtendAnalysisFarm, FarmLocation, FarmResolution, FarmSource,
|
|
139
|
+
export type { ExtendAnalysisFarm, FarmLocation, FarmResolution, FarmSource, PackageRequestOutcome, ResolveAnalysisFarm, ToolchainSource, } from "./sandbox/types.js";
|
|
140
|
+
export { formatQuery, identityAddress, identityKey, identityOf, parseIdentityKey, parseQuery, pythonIdentity, resolveQuery, rIdentity, } from "./sandbox/package-identity.js";
|
|
141
|
+
export type { PackageIdentity, PackageQuery, ParseQueryError, PoolIndex, QueryResolution, Track } from "./sandbox/package-identity.js";
|
|
140
142
|
export { FARM_LOCK_FILE, FarmLockSchema, readFarmLock, readFarmLockFile } from "./sandbox/farm.js";
|
|
141
143
|
export type { FarmLock, FarmLockError } from "./sandbox/farm.js";
|
|
142
144
|
export { IMAGE_PACKAGES_FILE } from "./sandbox/image-packages.js";
|
package/dist/index.js
CHANGED
|
@@ -246,6 +246,11 @@ export { createSandboxClient } from "./sandbox/create-sandbox.js";
|
|
|
246
246
|
// The throw of the client seam. An embedder matches the class and reads the
|
|
247
247
|
// typed variant on `error`, instead of a string match on the message.
|
|
248
248
|
export { describeSandboxError, SandboxFailure } from "./sandbox/sandbox-error.js";
|
|
249
|
+
// The identity of a package, the query that asks for one, and the resolution
|
|
250
|
+
// between the two. An embedder reads `by_name` of the dependency graph with
|
|
251
|
+
// these, thus the host lookup, the census, and the graph key never disagree on
|
|
252
|
+
// the identity of a name.
|
|
253
|
+
export { formatQuery, identityAddress, identityKey, identityOf, parseIdentityKey, parseQuery, pythonIdentity, resolveQuery, rIdentity, } from "./sandbox/package-identity.js";
|
|
249
254
|
// The farm contract: one `inflexa.lock` per farm. An embedder that composes
|
|
250
255
|
// farms writes this shape, and the mount gate reads it — one schema, no copy.
|
|
251
256
|
export { FARM_LOCK_FILE, FarmLockSchema, readFarmLock, readFarmLockFile } from "./sandbox/farm.js";
|
package/dist/lib/metrics.d.ts
CHANGED
|
@@ -2,17 +2,75 @@
|
|
|
2
2
|
* Custom OTel metrics for Cortex.
|
|
3
3
|
*
|
|
4
4
|
* Instruments:
|
|
5
|
-
* - cortex.
|
|
6
|
-
* - cortex.
|
|
5
|
+
* - cortex.run.completed{status, workflow} — counter, one per run that reached a terminal status
|
|
6
|
+
* - cortex.run.duration_ms{status, workflow} — histogram, start to terminal status
|
|
7
|
+
* - cortex.step.completed{status, agent_id} — counter, one per executed step that settled
|
|
8
|
+
* - cortex.step.duration_ms{status, agent_id} — histogram, step start to its terminal ledger write
|
|
9
|
+
* - cortex.artifact.reconcile.dropped{agent_id} — counter for missing manifest entries
|
|
10
|
+
* - cortex.artifact.reconcile.input_dropped{agent_id, reason} — counter for lineage input drops
|
|
7
11
|
*
|
|
8
|
-
*
|
|
9
|
-
*
|
|
12
|
+
* Every label is a bounded vocabulary: `status` is the terminal enum, `workflow`
|
|
13
|
+
* names the workflow family, `agent_id` has about 15 values. No instrument
|
|
14
|
+
* carries a run, step, exec, user, or organization id — a label with unbounded
|
|
15
|
+
* values multiplies the series count.
|
|
16
|
+
*
|
|
17
|
+
* A run or step outcome is recorded INSIDE the DBOS step that persists the same
|
|
18
|
+
* outcome to the ledger (or behind the ledger's own compare-and-set where the
|
|
19
|
+
* body has no such step). DBOS caches a completed step and does not re-run its
|
|
20
|
+
* body on recovery, so a replayed workflow records nothing twice.
|
|
10
21
|
*
|
|
11
22
|
* Instruments are lazy so the record site binds to whichever `MeterProvider`
|
|
12
23
|
* is globally registered at first use. The metrics API has no late-binding
|
|
13
24
|
* proxy: a meter taken at module load, before `initOtel` runs, would stay
|
|
14
25
|
* on the no-op provider for the life of the process.
|
|
15
26
|
*/
|
|
27
|
+
import type { StepExecutionStatus } from "../state/schema.js";
|
|
28
|
+
/** Terminal status of a run. A budget pause records as `canceled`, the same as the run ledger. */
|
|
29
|
+
export type RunOutcome = "completed" | "partial" | "failed" | "canceled";
|
|
30
|
+
/** The workflow family a run belongs to. */
|
|
31
|
+
export type RunWorkflow = "analysis" | "target_assessment" | "data_profile";
|
|
32
|
+
/** Terminal status of an executed step. A blocker is a failure to deliver, so `blocked` maps here to `failed`. */
|
|
33
|
+
export type StepOutcome = "completed" | "failed" | "canceled";
|
|
34
|
+
/**
|
|
35
|
+
* Bucket bounds for durations that run from seconds to hours: 30 s, then
|
|
36
|
+
* 1, 2, 5, 10, 30, 60, 120, 240 min. Ten buckets with the overflow, so one
|
|
37
|
+
* histogram costs about ten series per label set.
|
|
38
|
+
*/
|
|
39
|
+
export declare const RUN_DURATION_BOUNDS_MS: readonly number[];
|
|
40
|
+
/**
|
|
41
|
+
* Record one run that reached a terminal status. Call it inside the DBOS step
|
|
42
|
+
* that persists that status, or behind the ledger compare-and-set that accepted
|
|
43
|
+
* it. `durationMs` is omitted when the start instant is unknown: the run is
|
|
44
|
+
* then counted, not timed.
|
|
45
|
+
*/
|
|
46
|
+
export declare function recordRunCompleted(args: {
|
|
47
|
+
readonly workflow: RunWorkflow;
|
|
48
|
+
readonly status: RunOutcome;
|
|
49
|
+
readonly durationMs?: number;
|
|
50
|
+
}): void;
|
|
51
|
+
/**
|
|
52
|
+
* Record one executed step that settled. Call it inside the DBOS step that
|
|
53
|
+
* persists the terminal row, or inside the terminal step of the run for a
|
|
54
|
+
* step whose own body never reached one. `durationMs` is omitted when the
|
|
55
|
+
* step settled without a durable duration: it is then counted, not timed.
|
|
56
|
+
*/
|
|
57
|
+
export declare function recordStepCompleted(args: {
|
|
58
|
+
readonly agentId: string;
|
|
59
|
+
readonly status: StepOutcome;
|
|
60
|
+
readonly durationMs?: number;
|
|
61
|
+
}): void;
|
|
62
|
+
/**
|
|
63
|
+
* The step outcome that a terminal ledger status records. A status that is not
|
|
64
|
+
* terminal (`pending`, `running`) or that names a step that never executed
|
|
65
|
+
* (`skipped`) records no outcome.
|
|
66
|
+
*/
|
|
67
|
+
export declare function stepOutcomeOf(status: StepExecutionStatus): StepOutcome | undefined;
|
|
68
|
+
/**
|
|
69
|
+
* Milliseconds from a persisted ISO-8601 start instant to `nowMs`. `undefined`
|
|
70
|
+
* when the ledger holds no start or an unparsable one, so the caller counts
|
|
71
|
+
* without a duration instead of recording a bogus one.
|
|
72
|
+
*/
|
|
73
|
+
export declare function elapsedSinceIso(startedAt: string | null | undefined, nowMs?: number): number | undefined;
|
|
16
74
|
/** Record one manifest entry dropped at reconcile. */
|
|
17
75
|
export declare function recordArtifactReconcileDropped(args: {
|
|
18
76
|
readonly agentId: string;
|
|
@@ -24,4 +82,4 @@ export declare function recordLineageInputDropped(args: {
|
|
|
24
82
|
readonly reason: LineageInputDropReason;
|
|
25
83
|
}): void;
|
|
26
84
|
/** Test hook — drop memoised instruments to rebind a fresh MeterProvider. */
|
|
27
|
-
export declare function
|
|
85
|
+
export declare function __resetMetricsForTest(): void;
|
package/dist/lib/metrics.js
CHANGED
|
@@ -2,11 +2,22 @@
|
|
|
2
2
|
* Custom OTel metrics for Cortex.
|
|
3
3
|
*
|
|
4
4
|
* Instruments:
|
|
5
|
-
* - cortex.
|
|
6
|
-
* - cortex.
|
|
5
|
+
* - cortex.run.completed{status, workflow} — counter, one per run that reached a terminal status
|
|
6
|
+
* - cortex.run.duration_ms{status, workflow} — histogram, start to terminal status
|
|
7
|
+
* - cortex.step.completed{status, agent_id} — counter, one per executed step that settled
|
|
8
|
+
* - cortex.step.duration_ms{status, agent_id} — histogram, step start to its terminal ledger write
|
|
9
|
+
* - cortex.artifact.reconcile.dropped{agent_id} — counter for missing manifest entries
|
|
10
|
+
* - cortex.artifact.reconcile.input_dropped{agent_id, reason} — counter for lineage input drops
|
|
7
11
|
*
|
|
8
|
-
*
|
|
9
|
-
*
|
|
12
|
+
* Every label is a bounded vocabulary: `status` is the terminal enum, `workflow`
|
|
13
|
+
* names the workflow family, `agent_id` has about 15 values. No instrument
|
|
14
|
+
* carries a run, step, exec, user, or organization id — a label with unbounded
|
|
15
|
+
* values multiplies the series count.
|
|
16
|
+
*
|
|
17
|
+
* A run or step outcome is recorded INSIDE the DBOS step that persists the same
|
|
18
|
+
* outcome to the ledger (or behind the ledger's own compare-and-set where the
|
|
19
|
+
* body has no such step). DBOS caches a completed step and does not re-run its
|
|
20
|
+
* body on recovery, so a replayed workflow records nothing twice.
|
|
10
21
|
*
|
|
11
22
|
* Instruments are lazy so the record site binds to whichever `MeterProvider`
|
|
12
23
|
* is globally registered at first use. The metrics API has no late-binding
|
|
@@ -14,11 +25,36 @@
|
|
|
14
25
|
* on the no-op provider for the life of the process.
|
|
15
26
|
*/
|
|
16
27
|
import { metrics } from "@opentelemetry/api";
|
|
28
|
+
/**
|
|
29
|
+
* Bucket bounds for durations that run from seconds to hours: 30 s, then
|
|
30
|
+
* 1, 2, 5, 10, 30, 60, 120, 240 min. Ten buckets with the overflow, so one
|
|
31
|
+
* histogram costs about ten series per label set.
|
|
32
|
+
*/
|
|
33
|
+
export const RUN_DURATION_BOUNDS_MS = [30_000, 60_000, 120_000, 300_000, 600_000, 1_800_000, 3_600_000, 7_200_000, 14_400_000];
|
|
17
34
|
let instruments;
|
|
18
35
|
function getInstruments() {
|
|
19
36
|
if (instruments === undefined) {
|
|
20
37
|
const meter = metrics.getMeter("cortex");
|
|
38
|
+
const advice = { explicitBucketBoundaries: [...RUN_DURATION_BOUNDS_MS] };
|
|
21
39
|
instruments = {
|
|
40
|
+
runCompleted: meter.createCounter("cortex.run.completed", {
|
|
41
|
+
description: "Runs that reached a terminal status. Tagged by status, workflow.",
|
|
42
|
+
unit: "{run}",
|
|
43
|
+
}),
|
|
44
|
+
runDuration: meter.createHistogram("cortex.run.duration_ms", {
|
|
45
|
+
description: "Run duration from its start to its terminal status. Tagged by status, workflow.",
|
|
46
|
+
unit: "ms",
|
|
47
|
+
advice,
|
|
48
|
+
}),
|
|
49
|
+
stepCompleted: meter.createCounter("cortex.step.completed", {
|
|
50
|
+
description: "Executed steps that settled. Tagged by status, agent_id.",
|
|
51
|
+
unit: "{step}",
|
|
52
|
+
}),
|
|
53
|
+
stepDuration: meter.createHistogram("cortex.step.duration_ms", {
|
|
54
|
+
description: "Step duration from its start to its terminal ledger write. Tagged by status, agent_id.",
|
|
55
|
+
unit: "ms",
|
|
56
|
+
advice,
|
|
57
|
+
}),
|
|
22
58
|
artifactReconcileDropped: meter.createCounter("cortex.artifact.reconcile.dropped", {
|
|
23
59
|
description: "Manifest entries dropped because their on-disk file was missing at " +
|
|
24
60
|
"registration time (writes-then-deletes, renames). Tagged by agent_id.",
|
|
@@ -33,6 +69,69 @@ function getInstruments() {
|
|
|
33
69
|
}
|
|
34
70
|
return instruments;
|
|
35
71
|
}
|
|
72
|
+
/**
|
|
73
|
+
* Record one run that reached a terminal status. Call it inside the DBOS step
|
|
74
|
+
* that persists that status, or behind the ledger compare-and-set that accepted
|
|
75
|
+
* it. `durationMs` is omitted when the start instant is unknown: the run is
|
|
76
|
+
* then counted, not timed.
|
|
77
|
+
*/
|
|
78
|
+
export function recordRunCompleted(args) {
|
|
79
|
+
const attributes = { status: args.status, workflow: args.workflow };
|
|
80
|
+
const { runCompleted, runDuration } = getInstruments();
|
|
81
|
+
runCompleted.add(1, attributes);
|
|
82
|
+
if (args.durationMs !== undefined)
|
|
83
|
+
runDuration.record(Math.max(0, args.durationMs), attributes);
|
|
84
|
+
}
|
|
85
|
+
/**
|
|
86
|
+
* Record one executed step that settled. Call it inside the DBOS step that
|
|
87
|
+
* persists the terminal row, or inside the terminal step of the run for a
|
|
88
|
+
* step whose own body never reached one. `durationMs` is omitted when the
|
|
89
|
+
* step settled without a durable duration: it is then counted, not timed.
|
|
90
|
+
*/
|
|
91
|
+
export function recordStepCompleted(args) {
|
|
92
|
+
const attributes = { status: args.status, agent_id: args.agentId };
|
|
93
|
+
const { stepCompleted, stepDuration } = getInstruments();
|
|
94
|
+
stepCompleted.add(1, attributes);
|
|
95
|
+
if (args.durationMs !== undefined)
|
|
96
|
+
stepDuration.record(Math.max(0, args.durationMs), attributes);
|
|
97
|
+
}
|
|
98
|
+
/**
|
|
99
|
+
* The step outcome that a terminal ledger status records. A status that is not
|
|
100
|
+
* terminal (`pending`, `running`) or that names a step that never executed
|
|
101
|
+
* (`skipped`) records no outcome.
|
|
102
|
+
*/
|
|
103
|
+
export function stepOutcomeOf(status) {
|
|
104
|
+
switch (status) {
|
|
105
|
+
case "completed":
|
|
106
|
+
return "completed";
|
|
107
|
+
case "failed":
|
|
108
|
+
case "blocked":
|
|
109
|
+
return "failed";
|
|
110
|
+
case "canceled":
|
|
111
|
+
return "canceled";
|
|
112
|
+
case "pending":
|
|
113
|
+
case "running":
|
|
114
|
+
case "skipped":
|
|
115
|
+
return undefined;
|
|
116
|
+
default: {
|
|
117
|
+
const _exhaustive = status;
|
|
118
|
+
return _exhaustive;
|
|
119
|
+
}
|
|
120
|
+
}
|
|
121
|
+
}
|
|
122
|
+
/**
|
|
123
|
+
* Milliseconds from a persisted ISO-8601 start instant to `nowMs`. `undefined`
|
|
124
|
+
* when the ledger holds no start or an unparsable one, so the caller counts
|
|
125
|
+
* without a duration instead of recording a bogus one.
|
|
126
|
+
*/
|
|
127
|
+
export function elapsedSinceIso(startedAt, nowMs = Date.now()) {
|
|
128
|
+
if (startedAt === null || startedAt === undefined)
|
|
129
|
+
return undefined;
|
|
130
|
+
const startedMs = Date.parse(startedAt);
|
|
131
|
+
if (!Number.isFinite(startedMs))
|
|
132
|
+
return undefined;
|
|
133
|
+
return Math.max(0, nowMs - startedMs);
|
|
134
|
+
}
|
|
36
135
|
/** Record one manifest entry dropped at reconcile. */
|
|
37
136
|
export function recordArtifactReconcileDropped(args) {
|
|
38
137
|
getInstruments().artifactReconcileDropped.add(1, { agent_id: args.agentId });
|
|
@@ -42,6 +141,6 @@ export function recordLineageInputDropped(args) {
|
|
|
42
141
|
getInstruments().lineageInputDropped.add(1, { agent_id: args.agentId, reason: args.reason });
|
|
43
142
|
}
|
|
44
143
|
/** Test hook — drop memoised instruments to rebind a fresh MeterProvider. */
|
|
45
|
-
export function
|
|
144
|
+
export function __resetMetricsForTest() {
|
|
46
145
|
instruments = undefined;
|
|
47
146
|
}
|
|
@@ -0,0 +1,77 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Span policy of the harness TracerProvider: which DBOS traces are recorded,
|
|
3
|
+
* which spans reach the exporter, and the names the exported spans carry.
|
|
4
|
+
*
|
|
5
|
+
* DBOS names a workflow span after the registered workflow and a step span
|
|
6
|
+
* after the step function, and it runs each body under its span, so every
|
|
7
|
+
* child (a step, a patched fetch, a pg query) sits under the workflow span.
|
|
8
|
+
* Three shapes of that output are noise in a trace store, and each is stopped
|
|
9
|
+
* in the process, before export, rather than in a collector downstream. The
|
|
10
|
+
* decisions live where the workflows and steps are written; this module holds
|
|
11
|
+
* the mechanism only:
|
|
12
|
+
*
|
|
13
|
+
* - A module that registers a housekeeping workflow declares it untraced with
|
|
14
|
+
* `untracedWorkflow(name)` next to its registration. The sampler drops a
|
|
15
|
+
* root span that carries such a name at creation, and `ParentBased`
|
|
16
|
+
* sampling carries the decision to every child.
|
|
17
|
+
* - A replayed step re-emits its span with `cached=true`, which DBOS sets
|
|
18
|
+
* after the span has started. A sampler cannot see that, so
|
|
19
|
+
* `DbosSpanProcessor` drops the span at `onEnd`.
|
|
20
|
+
* - A step whose DBOS name embeds an id calls `stableSpan` as the first
|
|
21
|
+
* statement of its body. The helper renames the active DBOS span to the
|
|
22
|
+
* stable prefix and puts the id in an `inflexa.*` attribute. The DBOS step
|
|
23
|
+
* name itself is untouched: DBOS compares it against the recorded name on
|
|
24
|
+
* replay and throws `DBOSUnexpectedStepError` on a mismatch, so a rename
|
|
25
|
+
* would break every workflow in flight across a deploy.
|
|
26
|
+
*/
|
|
27
|
+
import { type Attributes, type Context } from "@opentelemetry/api";
|
|
28
|
+
import { type ReadableSpan, type Sampler, type Span, type SpanProcessor } from "@opentelemetry/sdk-trace-base";
|
|
29
|
+
/**
|
|
30
|
+
* Declare a registered DBOS workflow untraced: no root span for it is
|
|
31
|
+
* recorded, and no child span under it. Call it next to the
|
|
32
|
+
* `DBOS.registerWorkflow` that gives the workflow the same name.
|
|
33
|
+
*/
|
|
34
|
+
export declare function untracedWorkflow(name: string): void;
|
|
35
|
+
/**
|
|
36
|
+
* The sampler of the harness TracerProvider. A root span named after an
|
|
37
|
+
* untraced workflow is not recorded; every other root is. A span with a parent
|
|
38
|
+
* takes the parent's decision (the standard `ParentBased` behaviour), so a DBOS
|
|
39
|
+
* step, a fetch, or a pg span under a dropped root is dropped with it, and a
|
|
40
|
+
* remote parent (an inbound `traceparent`) is honoured as usual.
|
|
41
|
+
*/
|
|
42
|
+
export declare function createHarnessSampler(): Sampler;
|
|
43
|
+
/** The plan step id a `compose-step-seed` step carries. */
|
|
44
|
+
export declare const ATTR_INFLEXA_STEP_ID = "inflexa.step_id";
|
|
45
|
+
/** The provider tool-use id of a tool step. */
|
|
46
|
+
export declare const ATTR_INFLEXA_TOOL_USE_ID = "inflexa.tool_use_id";
|
|
47
|
+
/** The exec id (`${workflowId}:${stepId}:${fnId}`) of a sandbox exec step. */
|
|
48
|
+
export declare const ATTR_INFLEXA_EXEC_ID = "inflexa.exec_id";
|
|
49
|
+
/** The 1-based attempt counter of a sandbox poll, pull, or liveness-probe step. */
|
|
50
|
+
export declare const ATTR_INFLEXA_ATTEMPT = "inflexa.attempt";
|
|
51
|
+
/** The per-item key (a ChEMBL or NCT id) of a target-assessment fan-out step. */
|
|
52
|
+
export declare const ATTR_INFLEXA_FANOUT_KEY = "inflexa.fanout_key";
|
|
53
|
+
/**
|
|
54
|
+
* Rename the DBOS span of the step that is running to `name` and put its id
|
|
55
|
+
* in `attributes`. Call it as the first statement of a step body whose DBOS
|
|
56
|
+
* name embeds an id, with that DBOS name as `dbosName`.
|
|
57
|
+
*
|
|
58
|
+
* DBOS runs the body under its span, so with tracing on the active span is
|
|
59
|
+
* the step span. With tracing off DBOS opens no span, and the active span, if
|
|
60
|
+
* there is one, belongs to whoever started the workflow or the chat turn. The
|
|
61
|
+
* helper therefore renames a span only when its name is `dbosName`, and it
|
|
62
|
+
* leaves a missing or non-recording span alone.
|
|
63
|
+
*/
|
|
64
|
+
export declare function stableSpan(dbosName: string, name: string, attributes: Attributes): void;
|
|
65
|
+
/**
|
|
66
|
+
* Sits in front of the exporting processor and forwards every span except a
|
|
67
|
+
* replayed one (`cached=true`), which it drops at `onEnd`.
|
|
68
|
+
*/
|
|
69
|
+
export declare class DbosSpanProcessor implements SpanProcessor {
|
|
70
|
+
private readonly next;
|
|
71
|
+
constructor(next: SpanProcessor);
|
|
72
|
+
onStart(span: Span, parentContext: Context): void;
|
|
73
|
+
onEnding(span: Span): void;
|
|
74
|
+
onEnd(span: ReadableSpan): void;
|
|
75
|
+
forceFlush(): Promise<void>;
|
|
76
|
+
shutdown(): Promise<void>;
|
|
77
|
+
}
|
|
@@ -0,0 +1,123 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Span policy of the harness TracerProvider: which DBOS traces are recorded,
|
|
3
|
+
* which spans reach the exporter, and the names the exported spans carry.
|
|
4
|
+
*
|
|
5
|
+
* DBOS names a workflow span after the registered workflow and a step span
|
|
6
|
+
* after the step function, and it runs each body under its span, so every
|
|
7
|
+
* child (a step, a patched fetch, a pg query) sits under the workflow span.
|
|
8
|
+
* Three shapes of that output are noise in a trace store, and each is stopped
|
|
9
|
+
* in the process, before export, rather than in a collector downstream. The
|
|
10
|
+
* decisions live where the workflows and steps are written; this module holds
|
|
11
|
+
* the mechanism only:
|
|
12
|
+
*
|
|
13
|
+
* - A module that registers a housekeeping workflow declares it untraced with
|
|
14
|
+
* `untracedWorkflow(name)` next to its registration. The sampler drops a
|
|
15
|
+
* root span that carries such a name at creation, and `ParentBased`
|
|
16
|
+
* sampling carries the decision to every child.
|
|
17
|
+
* - A replayed step re-emits its span with `cached=true`, which DBOS sets
|
|
18
|
+
* after the span has started. A sampler cannot see that, so
|
|
19
|
+
* `DbosSpanProcessor` drops the span at `onEnd`.
|
|
20
|
+
* - A step whose DBOS name embeds an id calls `stableSpan` as the first
|
|
21
|
+
* statement of its body. The helper renames the active DBOS span to the
|
|
22
|
+
* stable prefix and puts the id in an `inflexa.*` attribute. The DBOS step
|
|
23
|
+
* name itself is untouched: DBOS compares it against the recorded name on
|
|
24
|
+
* replay and throws `DBOSUnexpectedStepError` on a mismatch, so a rename
|
|
25
|
+
* would break every workflow in flight across a deploy.
|
|
26
|
+
*/
|
|
27
|
+
import { trace } from "@opentelemetry/api";
|
|
28
|
+
import { ParentBasedSampler, SamplingDecision, } from "@opentelemetry/sdk-trace-base";
|
|
29
|
+
/**
|
|
30
|
+
* Names of the DBOS workflows whose traces are not recorded. A registering
|
|
31
|
+
* module adds its own name at registration, which runs at boot inside
|
|
32
|
+
* `assembleCoreRuntime`, before `DBOS.launch` starts the first workflow. The
|
|
33
|
+
* sampler reads the set on every root span, so the set is complete before it
|
|
34
|
+
* is consulted, and a name added later applies from the next root span on.
|
|
35
|
+
*/
|
|
36
|
+
const untracedWorkflowNames = new Set();
|
|
37
|
+
/**
|
|
38
|
+
* Declare a registered DBOS workflow untraced: no root span for it is
|
|
39
|
+
* recorded, and no child span under it. Call it next to the
|
|
40
|
+
* `DBOS.registerWorkflow` that gives the workflow the same name.
|
|
41
|
+
*/
|
|
42
|
+
export function untracedWorkflow(name) {
|
|
43
|
+
untracedWorkflowNames.add(name);
|
|
44
|
+
}
|
|
45
|
+
/** Root decision only: `NOT_RECORD` for an untraced workflow, `RECORD_AND_SAMPLED` for every other root. */
|
|
46
|
+
class UntracedWorkflowRootSampler {
|
|
47
|
+
shouldSample(_context, _traceId, spanName) {
|
|
48
|
+
return {
|
|
49
|
+
decision: untracedWorkflowNames.has(spanName) ? SamplingDecision.NOT_RECORD : SamplingDecision.RECORD_AND_SAMPLED,
|
|
50
|
+
};
|
|
51
|
+
}
|
|
52
|
+
toString() {
|
|
53
|
+
return "UntracedWorkflowRootSampler";
|
|
54
|
+
}
|
|
55
|
+
}
|
|
56
|
+
/**
|
|
57
|
+
* The sampler of the harness TracerProvider. A root span named after an
|
|
58
|
+
* untraced workflow is not recorded; every other root is. A span with a parent
|
|
59
|
+
* takes the parent's decision (the standard `ParentBased` behaviour), so a DBOS
|
|
60
|
+
* step, a fetch, or a pg span under a dropped root is dropped with it, and a
|
|
61
|
+
* remote parent (an inbound `traceparent`) is honoured as usual.
|
|
62
|
+
*/
|
|
63
|
+
export function createHarnessSampler() {
|
|
64
|
+
return new ParentBasedSampler({ root: new UntracedWorkflowRootSampler() });
|
|
65
|
+
}
|
|
66
|
+
/** The plan step id a `compose-step-seed` step carries. */
|
|
67
|
+
export const ATTR_INFLEXA_STEP_ID = "inflexa.step_id";
|
|
68
|
+
/** The provider tool-use id of a tool step. */
|
|
69
|
+
export const ATTR_INFLEXA_TOOL_USE_ID = "inflexa.tool_use_id";
|
|
70
|
+
/** The exec id (`${workflowId}:${stepId}:${fnId}`) of a sandbox exec step. */
|
|
71
|
+
export const ATTR_INFLEXA_EXEC_ID = "inflexa.exec_id";
|
|
72
|
+
/** The 1-based attempt counter of a sandbox poll, pull, or liveness-probe step. */
|
|
73
|
+
export const ATTR_INFLEXA_ATTEMPT = "inflexa.attempt";
|
|
74
|
+
/** The per-item key (a ChEMBL or NCT id) of a target-assessment fan-out step. */
|
|
75
|
+
export const ATTR_INFLEXA_FANOUT_KEY = "inflexa.fanout_key";
|
|
76
|
+
/**
|
|
77
|
+
* Rename the DBOS span of the step that is running to `name` and put its id
|
|
78
|
+
* in `attributes`. Call it as the first statement of a step body whose DBOS
|
|
79
|
+
* name embeds an id, with that DBOS name as `dbosName`.
|
|
80
|
+
*
|
|
81
|
+
* DBOS runs the body under its span, so with tracing on the active span is
|
|
82
|
+
* the step span. With tracing off DBOS opens no span, and the active span, if
|
|
83
|
+
* there is one, belongs to whoever started the workflow or the chat turn. The
|
|
84
|
+
* helper therefore renames a span only when its name is `dbosName`, and it
|
|
85
|
+
* leaves a missing or non-recording span alone.
|
|
86
|
+
*/
|
|
87
|
+
export function stableSpan(dbosName, name, attributes) {
|
|
88
|
+
const span = trace.getActiveSpan();
|
|
89
|
+
if (span === undefined || !span.isRecording())
|
|
90
|
+
return;
|
|
91
|
+
// The API `Span` has no name accessor; the SDK span the provider hands out does.
|
|
92
|
+
if (span.name !== dbosName)
|
|
93
|
+
return;
|
|
94
|
+
span.updateName(name);
|
|
95
|
+
span.setAttributes(attributes);
|
|
96
|
+
}
|
|
97
|
+
/**
|
|
98
|
+
* Sits in front of the exporting processor and forwards every span except a
|
|
99
|
+
* replayed one (`cached=true`), which it drops at `onEnd`.
|
|
100
|
+
*/
|
|
101
|
+
export class DbosSpanProcessor {
|
|
102
|
+
next;
|
|
103
|
+
constructor(next) {
|
|
104
|
+
this.next = next;
|
|
105
|
+
}
|
|
106
|
+
onStart(span, parentContext) {
|
|
107
|
+
this.next.onStart(span, parentContext);
|
|
108
|
+
}
|
|
109
|
+
onEnding(span) {
|
|
110
|
+
this.next.onEnding?.(span);
|
|
111
|
+
}
|
|
112
|
+
onEnd(span) {
|
|
113
|
+
if (span.attributes.cached === true)
|
|
114
|
+
return;
|
|
115
|
+
this.next.onEnd(span);
|
|
116
|
+
}
|
|
117
|
+
forceFlush() {
|
|
118
|
+
return this.next.forceFlush();
|
|
119
|
+
}
|
|
120
|
+
shutdown() {
|
|
121
|
+
return this.next.shutdown();
|
|
122
|
+
}
|
|
123
|
+
}
|