@untangle-code/jobs 0.0.0-stage → 0.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +661 -0
- package/NOTICE +90 -0
- package/README.md +138 -2
- package/THIRD_PARTY_LICENSES.md +24 -0
- package/dist/abort.d.ts +14 -0
- package/dist/abort.d.ts.map +1 -0
- package/dist/abort.js +35 -0
- package/dist/abort.js.map +1 -0
- package/dist/bounded-queue.d.ts +200 -0
- package/dist/bounded-queue.d.ts.map +1 -0
- package/dist/bounded-queue.js +338 -0
- package/dist/bounded-queue.js.map +1 -0
- package/dist/checkpoint-store.d.ts +30 -0
- package/dist/checkpoint-store.d.ts.map +1 -0
- package/dist/checkpoint-store.js +131 -0
- package/dist/checkpoint-store.js.map +1 -0
- package/dist/events.d.ts +29 -0
- package/dist/events.d.ts.map +1 -0
- package/dist/events.js +66 -0
- package/dist/events.js.map +1 -0
- package/dist/index.d.ts +22 -0
- package/dist/index.d.ts.map +1 -0
- package/dist/index.js +30 -0
- package/dist/index.js.map +1 -0
- package/dist/resume.d.ts +45 -0
- package/dist/resume.d.ts.map +1 -0
- package/dist/resume.js +60 -0
- package/dist/resume.js.map +1 -0
- package/dist/runner.d.ts +148 -0
- package/dist/runner.d.ts.map +1 -0
- package/dist/runner.js +686 -0
- package/dist/runner.js.map +1 -0
- package/dist/stages.d.ts +146 -0
- package/dist/stages.d.ts.map +1 -0
- package/dist/stages.js +2 -0
- package/dist/stages.js.map +1 -0
- package/dist/state-machine.d.ts +52 -0
- package/dist/state-machine.d.ts.map +1 -0
- package/dist/state-machine.js +114 -0
- package/dist/state-machine.js.map +1 -0
- package/package.json +61 -3
- package/src/abort.ts +37 -0
- package/src/bounded-queue.ts +462 -0
- package/src/checkpoint-store.ts +171 -0
- package/src/events.ts +71 -0
- package/src/index.ts +37 -0
- package/src/resume.ts +106 -0
- package/src/runner.ts +916 -0
- package/src/stages.ts +167 -0
- package/src/state-machine.ts +132 -0
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;GAYG;AAGH,cAAc,YAAY,CAAC;AAG3B,cAAc,oBAAoB,CAAC;AAGnC,cAAc,aAAa,CAAC;AAG5B,cAAc,oBAAoB,CAAC;AAGnC,cAAc,aAAa,CAAC;AAG5B,cAAc,uBAAuB,CAAC;AAGtC,cAAc,aAAa,CAAC;AAG5B,cAAc,aAAa,CAAC"}
|
package/dist/index.js
ADDED
|
@@ -0,0 +1,30 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* @untangle-code/jobs — the job core of the Untangle indexer
|
|
3
|
+
* (spec.md §11, §12.2; architecture.md §2.2, §6.14, §13, §21.3–§21.4).
|
|
4
|
+
*
|
|
5
|
+
* Pipeline MECHANICS only: the exact spec.md §11.1 state machine, typed job
|
|
6
|
+
* events as `AsyncIterable`, a checkpoint store (`IndexCheckpointV1`,
|
|
7
|
+
* checkpoints only after confirmed upserts), resume planning, generation
|
|
8
|
+
* commit per §13.4, bounded queues with backpressure and a visible memory
|
|
9
|
+
* budget, and `AbortSignal` cancellation at safe batch boundaries. The
|
|
10
|
+
* domain-specific stages (scanner, parser, chunker, embedder, stores) are
|
|
11
|
+
* injected behind narrow interfaces. This module is the ONLY public export
|
|
12
|
+
* of the package.
|
|
13
|
+
*/
|
|
14
|
+
// Abort helpers (spec.md §22).
|
|
15
|
+
export * from "./abort.js";
|
|
16
|
+
// State machine (spec.md §11.1).
|
|
17
|
+
export * from "./state-machine.js";
|
|
18
|
+
// Async event channel (architecture.md §2.2).
|
|
19
|
+
export * from "./events.js";
|
|
20
|
+
// Bounded queues with backpressure (architecture.md §13.2).
|
|
21
|
+
export * from "./bounded-queue.js";
|
|
22
|
+
// Injected stage interfaces (architecture.md §13.1).
|
|
23
|
+
export * from "./stages.js";
|
|
24
|
+
// Checkpoint store (architecture.md §13.3; spec.md §10.5).
|
|
25
|
+
export * from "./checkpoint-store.js";
|
|
26
|
+
// Resume planning (spec.md §11.2).
|
|
27
|
+
export * from "./resume.js";
|
|
28
|
+
// Index job runner (spec.md §11; architecture.md §13.4, §21.3).
|
|
29
|
+
export * from "./runner.js";
|
|
30
|
+
//# sourceMappingURL=index.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"index.js","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;GAYG;AAEH,+BAA+B;AAC/B,cAAc,YAAY,CAAC;AAE3B,iCAAiC;AACjC,cAAc,oBAAoB,CAAC;AAEnC,8CAA8C;AAC9C,cAAc,aAAa,CAAC;AAE5B,4DAA4D;AAC5D,cAAc,oBAAoB,CAAC;AAEnC,qDAAqD;AACrD,cAAc,aAAa,CAAC;AAE5B,2DAA2D;AAC3D,cAAc,uBAAuB,CAAC;AAEtC,mCAAmC;AACnC,cAAc,aAAa,CAAC;AAE5B,gEAAgE;AAChE,cAAc,aAAa,CAAC"}
|
package/dist/resume.d.ts
ADDED
|
@@ -0,0 +1,45 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Resume planning (architecture.md §6.14): reconstruct the remaining work
|
|
3
|
+
* plan from the last CONFIRMED checkpoint, so a restarted job continues
|
|
4
|
+
* exactly where confirmed work ended (spec.md §11.2) and never repeats an
|
|
5
|
+
* already-confirmed file.
|
|
6
|
+
*/
|
|
7
|
+
import type { IndexCheckpointV1 } from "@untangle-code/domain";
|
|
8
|
+
import type { PlannedFileV1 } from "./stages.js";
|
|
9
|
+
/** The identity a checkpoint must match to be resumable for this job. */
|
|
10
|
+
export interface ResumeIdentityV1 {
|
|
11
|
+
workspaceId: string;
|
|
12
|
+
repoId: string;
|
|
13
|
+
refId: string;
|
|
14
|
+
effectiveProfileId: string;
|
|
15
|
+
}
|
|
16
|
+
/**
|
|
17
|
+
* True when `checkpoint` belongs to exactly this workspace/repo/ref/profile.
|
|
18
|
+
* A profile mismatch means an incompatible embedding profile — such a
|
|
19
|
+
* checkpoint MUST NOT be resumed; the job needs its own new generation
|
|
20
|
+
* (spec.md §11.2).
|
|
21
|
+
*/
|
|
22
|
+
export declare function checkpointMatches(checkpoint: IndexCheckpointV1, identity: ResumeIdentityV1): boolean;
|
|
23
|
+
export interface ResumePlanV1 {
|
|
24
|
+
/** True when a matching checkpoint was adopted. */
|
|
25
|
+
resumed: boolean;
|
|
26
|
+
/** Generation to continue (from the checkpoint), or null for a fresh run. */
|
|
27
|
+
generationId: string | null;
|
|
28
|
+
/** Files that still need the full pipeline. */
|
|
29
|
+
remaining: PlannedFileV1[];
|
|
30
|
+
/** Confirmed state carried over from the checkpoint (planned ∩ unchanged). */
|
|
31
|
+
completedFiles: string[];
|
|
32
|
+
storedPointIdsByFile: Record<string, string[]>;
|
|
33
|
+
manifestHashes: Record<string, string>;
|
|
34
|
+
/** How many planned files were skipped because they are already confirmed. */
|
|
35
|
+
skippedCompleted: number;
|
|
36
|
+
}
|
|
37
|
+
/**
|
|
38
|
+
* Builds the remaining work plan. A planned file is skipped only when the
|
|
39
|
+
* checkpoint confirms it AND its content hash is unchanged — the content
|
|
40
|
+
* hash is the source of truth (spec.md §12.1). Confirmed files whose hash
|
|
41
|
+
* changed (or that are no longer planned) are dropped from the carried
|
|
42
|
+
* state so they are re-confirmed by this run, never counted twice.
|
|
43
|
+
*/
|
|
44
|
+
export declare function planResume(planned: readonly PlannedFileV1[], checkpoint: IndexCheckpointV1 | null, identity: ResumeIdentityV1): ResumePlanV1;
|
|
45
|
+
//# sourceMappingURL=resume.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"resume.d.ts","sourceRoot":"","sources":["../src/resume.ts"],"names":[],"mappings":"AAAA;;;;;GAKG;AACH,OAAO,KAAK,EAAE,iBAAiB,EAAE,MAAM,uBAAuB,CAAC;AAE/D,OAAO,KAAK,EAAE,aAAa,EAAE,MAAM,aAAa,CAAC;AAEjD,yEAAyE;AACzE,MAAM,WAAW,gBAAgB;IAC/B,WAAW,EAAE,MAAM,CAAC;IACpB,MAAM,EAAE,MAAM,CAAC;IACf,KAAK,EAAE,MAAM,CAAC;IACd,kBAAkB,EAAE,MAAM,CAAC;CAC5B;AAED;;;;;GAKG;AACH,wBAAgB,iBAAiB,CAC/B,UAAU,EAAE,iBAAiB,EAC7B,QAAQ,EAAE,gBAAgB,GACzB,OAAO,CAOT;AAED,MAAM,WAAW,YAAY;IAC3B,mDAAmD;IACnD,OAAO,EAAE,OAAO,CAAC;IACjB,6EAA6E;IAC7E,YAAY,EAAE,MAAM,GAAG,IAAI,CAAC;IAC5B,+CAA+C;IAC/C,SAAS,EAAE,aAAa,EAAE,CAAC;IAC3B,8EAA8E;IAC9E,cAAc,EAAE,MAAM,EAAE,CAAC;IACzB,oBAAoB,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,EAAE,CAAC,CAAC;IAC/C,cAAc,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;IACvC,8EAA8E;IAC9E,gBAAgB,EAAE,MAAM,CAAC;CAC1B;AAED;;;;;;GAMG;AACH,wBAAgB,UAAU,CACxB,OAAO,EAAE,SAAS,aAAa,EAAE,EACjC,UAAU,EAAE,iBAAiB,GAAG,IAAI,EACpC,QAAQ,EAAE,gBAAgB,GACzB,YAAY,CA2Cd"}
|
package/dist/resume.js
ADDED
|
@@ -0,0 +1,60 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* True when `checkpoint` belongs to exactly this workspace/repo/ref/profile.
|
|
3
|
+
* A profile mismatch means an incompatible embedding profile — such a
|
|
4
|
+
* checkpoint MUST NOT be resumed; the job needs its own new generation
|
|
5
|
+
* (spec.md §11.2).
|
|
6
|
+
*/
|
|
7
|
+
export function checkpointMatches(checkpoint, identity) {
|
|
8
|
+
return (checkpoint.workspaceId === identity.workspaceId &&
|
|
9
|
+
checkpoint.repoId === identity.repoId &&
|
|
10
|
+
checkpoint.refId === identity.refId &&
|
|
11
|
+
checkpoint.effectiveProfileId === identity.effectiveProfileId);
|
|
12
|
+
}
|
|
13
|
+
/**
|
|
14
|
+
* Builds the remaining work plan. A planned file is skipped only when the
|
|
15
|
+
* checkpoint confirms it AND its content hash is unchanged — the content
|
|
16
|
+
* hash is the source of truth (spec.md §12.1). Confirmed files whose hash
|
|
17
|
+
* changed (or that are no longer planned) are dropped from the carried
|
|
18
|
+
* state so they are re-confirmed by this run, never counted twice.
|
|
19
|
+
*/
|
|
20
|
+
export function planResume(planned, checkpoint, identity) {
|
|
21
|
+
if (checkpoint === null || !checkpointMatches(checkpoint, identity)) {
|
|
22
|
+
return {
|
|
23
|
+
resumed: false,
|
|
24
|
+
generationId: null,
|
|
25
|
+
remaining: [...planned],
|
|
26
|
+
completedFiles: [],
|
|
27
|
+
storedPointIdsByFile: {},
|
|
28
|
+
manifestHashes: {},
|
|
29
|
+
skippedCompleted: 0,
|
|
30
|
+
};
|
|
31
|
+
}
|
|
32
|
+
const confirmed = new Set(checkpoint.completedFiles);
|
|
33
|
+
const remaining = [];
|
|
34
|
+
const completedFiles = [];
|
|
35
|
+
const storedPointIdsByFile = {};
|
|
36
|
+
const manifestHashes = {};
|
|
37
|
+
for (const file of planned) {
|
|
38
|
+
const isConfirmedUnchanged = confirmed.has(file.fileId) && checkpoint.manifestHashes[file.fileId] === file.contentHash;
|
|
39
|
+
if (!isConfirmedUnchanged) {
|
|
40
|
+
remaining.push(file);
|
|
41
|
+
continue;
|
|
42
|
+
}
|
|
43
|
+
completedFiles.push(file.fileId);
|
|
44
|
+
manifestHashes[file.fileId] = file.contentHash;
|
|
45
|
+
const pointIds = checkpoint.storedPointIdsByFile[file.fileId];
|
|
46
|
+
if (pointIds !== undefined) {
|
|
47
|
+
storedPointIdsByFile[file.fileId] = [...pointIds];
|
|
48
|
+
}
|
|
49
|
+
}
|
|
50
|
+
return {
|
|
51
|
+
resumed: true,
|
|
52
|
+
generationId: checkpoint.generationId,
|
|
53
|
+
remaining,
|
|
54
|
+
completedFiles,
|
|
55
|
+
storedPointIdsByFile,
|
|
56
|
+
manifestHashes,
|
|
57
|
+
skippedCompleted: completedFiles.length,
|
|
58
|
+
};
|
|
59
|
+
}
|
|
60
|
+
//# sourceMappingURL=resume.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"resume.js","sourceRoot":"","sources":["../src/resume.ts"],"names":[],"mappings":"AAkBA;;;;;GAKG;AACH,MAAM,UAAU,iBAAiB,CAC/B,UAA6B,EAC7B,QAA0B;IAE1B,OAAO,CACL,UAAU,CAAC,WAAW,KAAK,QAAQ,CAAC,WAAW;QAC/C,UAAU,CAAC,MAAM,KAAK,QAAQ,CAAC,MAAM;QACrC,UAAU,CAAC,KAAK,KAAK,QAAQ,CAAC,KAAK;QACnC,UAAU,CAAC,kBAAkB,KAAK,QAAQ,CAAC,kBAAkB,CAC9D,CAAC;AACJ,CAAC;AAiBD;;;;;;GAMG;AACH,MAAM,UAAU,UAAU,CACxB,OAAiC,EACjC,UAAoC,EACpC,QAA0B;IAE1B,IAAI,UAAU,KAAK,IAAI,IAAI,CAAC,iBAAiB,CAAC,UAAU,EAAE,QAAQ,CAAC,EAAE,CAAC;QACpE,OAAO;YACL,OAAO,EAAE,KAAK;YACd,YAAY,EAAE,IAAI;YAClB,SAAS,EAAE,CAAC,GAAG,OAAO,CAAC;YACvB,cAAc,EAAE,EAAE;YAClB,oBAAoB,EAAE,EAAE;YACxB,cAAc,EAAE,EAAE;YAClB,gBAAgB,EAAE,CAAC;SACpB,CAAC;IACJ,CAAC;IAED,MAAM,SAAS,GAAG,IAAI,GAAG,CAAC,UAAU,CAAC,cAAc,CAAC,CAAC;IACrD,MAAM,SAAS,GAAoB,EAAE,CAAC;IACtC,MAAM,cAAc,GAAa,EAAE,CAAC;IACpC,MAAM,oBAAoB,GAA6B,EAAE,CAAC;IAC1D,MAAM,cAAc,GAA2B,EAAE,CAAC;IAElD,KAAK,MAAM,IAAI,IAAI,OAAO,EAAE,CAAC;QAC3B,MAAM,oBAAoB,GACxB,SAAS,CAAC,GAAG,CAAC,IAAI,CAAC,MAAM,CAAC,IAAI,UAAU,CAAC,cAAc,CAAC,IAAI,CAAC,MAAM,CAAC,KAAK,IAAI,CAAC,WAAW,CAAC;QAC5F,IAAI,CAAC,oBAAoB,EAAE,CAAC;YAC1B,SAAS,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;YACrB,SAAS;QACX,CAAC;QACD,cAAc,CAAC,IAAI,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;QACjC,cAAc,CAAC,IAAI,CAAC,MAAM,CAAC,GAAG,IAAI,CAAC,WAAW,CAAC;QAC/C,MAAM,QAAQ,GAAG,UAAU,CAAC,oBAAoB,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;QAC9D,IAAI,QAAQ,KAAK,SAAS,EAAE,CAAC;YAC3B,oBAAoB,CAAC,IAAI,CAAC,MAAM,CAAC,GAAG,CAAC,GAAG,QAAQ,CAAC,CAAC;QACpD,CAAC;IACH,CAAC;IAED,OAAO;QACL,OAAO,EAAE,IAAI;QACb,YAAY,EAAE,UAAU,CAAC,YAAY;QACrC,SAAS;QACT,cAAc;QACd,oBAAoB;QACpB,cAAc;QACd,gBAAgB,EAAE,cAAc,CAAC,MAAM;KACxC,CAAC;AACJ,CAAC"}
|
package/dist/runner.d.ts
ADDED
|
@@ -0,0 +1,148 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Index job runner — the pipeline MECHANICS of spec.md §11 and
|
|
3
|
+
* architecture.md §13:
|
|
4
|
+
*
|
|
5
|
+
* - drives the exact state machine of spec.md §11.1 (`JobStateMachine`);
|
|
6
|
+
* - emits the typed job events of architecture.md §2.2 as an
|
|
7
|
+
* `AsyncIterable<JobEvent>` (spec.md §22);
|
|
8
|
+
* - connects the injected stages with bounded queues so a full queue pauses
|
|
9
|
+
* the previous stage (backpressure, §13.2) under a configurable, visible
|
|
10
|
+
* memory budget;
|
|
11
|
+
* - persists an `IndexCheckpointV1` ONLY after a confirmed upsert
|
|
12
|
+
* (spec.md §10.5, §13.3), one per file batch (spec.md §11.2);
|
|
13
|
+
* - resumes from the last confirmed checkpoint without repeating confirmed
|
|
14
|
+
* work (spec.md §11.2);
|
|
15
|
+
* - cancels via `AbortSignal` at safe batch boundaries only: checkpoint
|
|
16
|
+
* saved, then `cancelled` (architecture.md §21.3);
|
|
17
|
+
* - commits the generation per §13.4 — `completed` is only reported when
|
|
18
|
+
* all planned file operations are confirmed, checkpoints are saved, the
|
|
19
|
+
* required graphs are stored and the metadata reached its final status
|
|
20
|
+
* (enforced through domain's `assertGenerationComplete`).
|
|
21
|
+
*/
|
|
22
|
+
import type { CheckpointStorePort, Clock, IndexJobStage, JobEvent, LockProvider, LoggerPort, Random } from "@untangle-code/domain";
|
|
23
|
+
import { type QueueStatsV1 } from "./bounded-queue.js";
|
|
24
|
+
import type { ResumeIdentityV1 } from "./resume.js";
|
|
25
|
+
import type { IndexJobStages } from "./stages.js";
|
|
26
|
+
/** Identity of the ref this job indexes. */
|
|
27
|
+
export type IndexJobIdentityV1 = ResumeIdentityV1;
|
|
28
|
+
export interface IndexJobQueueCapacities {
|
|
29
|
+
/** Analyzed-file queue capacity (items). Default 16. */
|
|
30
|
+
analysis?: number;
|
|
31
|
+
/** Embedded-batch queue capacity (items). Default 4. */
|
|
32
|
+
upsert?: number;
|
|
33
|
+
}
|
|
34
|
+
export interface IndexJobRunnerOptions {
|
|
35
|
+
identity: IndexJobIdentityV1;
|
|
36
|
+
stages: IndexJobStages;
|
|
37
|
+
checkpointStore: CheckpointStorePort;
|
|
38
|
+
clock: Clock;
|
|
39
|
+
random: Random;
|
|
40
|
+
logger?: LoggerPort;
|
|
41
|
+
/** When given, the job holds `repo:<repoId>:<refId>:index` while running. */
|
|
42
|
+
lockProvider?: LockProvider;
|
|
43
|
+
/**
|
|
44
|
+
* Duration of the index-lock lease. While the job is active, the runner
|
|
45
|
+
* renews it with a non-overlapping heartbeat at most one third this long.
|
|
46
|
+
* Default 30 seconds; must be a safe integer >= 3 milliseconds.
|
|
47
|
+
*/
|
|
48
|
+
lockLeaseMs?: number;
|
|
49
|
+
/** Job id; defaults to `random.uuid()`. */
|
|
50
|
+
jobId?: string;
|
|
51
|
+
/** Generation id for a FRESH run; a resumed run keeps the checkpoint's. */
|
|
52
|
+
generationId?: string;
|
|
53
|
+
/**
|
|
54
|
+
* Awaited immediately after the generation is chosen and before any
|
|
55
|
+
* analysis, graph, checkpoint, or index mutation starts. Adapters use this
|
|
56
|
+
* boundary to persist a durable `running` marker; rejection fails the job
|
|
57
|
+
* before the pipeline can mutate the index.
|
|
58
|
+
*/
|
|
59
|
+
onGenerationStarted?: (generationId: string) => Promise<void>;
|
|
60
|
+
/**
|
|
61
|
+
* Runs for a terminal outcome only after a final owner-checked lease renewal
|
|
62
|
+
* and before release. A detected ownership loss suppresses it. Adapters should serialize and
|
|
63
|
+
* generation-CAS any repo-wide singleton state they replace. Completed
|
|
64
|
+
* outcomes include an immutable fingerprint of the full discovered corpus;
|
|
65
|
+
* terminal failures deliberately do not.
|
|
66
|
+
*/
|
|
67
|
+
onGenerationFinished?: (generationId: string, outcome: "completed" | "cancelled" | "failed", corpusRevisionFingerprint?: string) => Promise<void>;
|
|
68
|
+
/** Graphs the generation must store to complete. Default ["file","symbol"]. */
|
|
69
|
+
requiredGraphs?: readonly ("file" | "symbol")[];
|
|
70
|
+
/** Files per embedding batch (also the checkpoint batch). Default 8. */
|
|
71
|
+
embedBatchSize?: number;
|
|
72
|
+
queueCapacities?: IndexJobQueueCapacities;
|
|
73
|
+
/**
|
|
74
|
+
* Byte budget shared by the pipeline queues (architecture.md §13.2),
|
|
75
|
+
* split evenly between them; visible in `status()`. Default 64 MiB.
|
|
76
|
+
*/
|
|
77
|
+
memoryBudgetBytes?: number;
|
|
78
|
+
/** Set false to ignore existing checkpoints and start fresh. Default true. */
|
|
79
|
+
resume?: boolean;
|
|
80
|
+
}
|
|
81
|
+
/** Live job status (architecture.md §13.2: budget visible in status output). */
|
|
82
|
+
export interface JobStatusV1 {
|
|
83
|
+
jobId: string;
|
|
84
|
+
generationId: string | null;
|
|
85
|
+
stage: IndexJobStage;
|
|
86
|
+
filesPlanned: number;
|
|
87
|
+
filesCompleted: number;
|
|
88
|
+
memoryBudgetBytes: number;
|
|
89
|
+
queuedBytes: number;
|
|
90
|
+
queues: {
|
|
91
|
+
analysis: QueueStatsV1;
|
|
92
|
+
upsert: QueueStatsV1;
|
|
93
|
+
} | null;
|
|
94
|
+
}
|
|
95
|
+
export declare class IndexJobRunner {
|
|
96
|
+
private readonly options;
|
|
97
|
+
private readonly machine;
|
|
98
|
+
private readonly jobId;
|
|
99
|
+
private generationId;
|
|
100
|
+
private readonly completedFiles;
|
|
101
|
+
private storedPointIdsByFile;
|
|
102
|
+
private manifestHashes;
|
|
103
|
+
private corpusManifestHashes;
|
|
104
|
+
private filesPlanned;
|
|
105
|
+
private queues;
|
|
106
|
+
private lastQueueStats;
|
|
107
|
+
private started;
|
|
108
|
+
constructor(options: IndexJobRunnerOptions);
|
|
109
|
+
/**
|
|
110
|
+
* Prueft die Ressourcen-Optionen am EINGANG des Jobs — dort, wo sie ein
|
|
111
|
+
* Aufrufer setzt, nicht erst tief in der Pipeline.
|
|
112
|
+
*
|
|
113
|
+
* WP-116: `queueCapacities` und `memoryBudgetBytes` sind frei
|
|
114
|
+
* konfigurierbar und entscheiden zusammen darueber, ob die Byte-Grenze der
|
|
115
|
+
* Queues ueberhaupt greifen kann. Die Haertung aus WP-79 sitzt im
|
|
116
|
+
* `BoundedQueue`-Konstruktor und damit erst in `runPipeline()` — ein
|
|
117
|
+
* fehlerhafter Wert faellt dort mitten im Lauf auf, nachdem der Job
|
|
118
|
+
* bereits Lock, Generation und Checkpoints angefasst hat. Hier scheitert
|
|
119
|
+
* er, bevor irgendetwas passiert ist.
|
|
120
|
+
*/
|
|
121
|
+
private static validateResourceOptions;
|
|
122
|
+
status(): JobStatusV1;
|
|
123
|
+
/**
|
|
124
|
+
* Runs the job once. Cold: work starts when iteration starts. The stream
|
|
125
|
+
* always ends with exactly one terminal event — `job_completed`,
|
|
126
|
+
* `job_failed` or `job_cancelled` — and never throws for job-level
|
|
127
|
+
* failures (they ARE the `job_failed` event).
|
|
128
|
+
*/
|
|
129
|
+
run(options?: {
|
|
130
|
+
signal?: AbortSignal;
|
|
131
|
+
}): AsyncIterable<JobEvent>;
|
|
132
|
+
private drive;
|
|
133
|
+
/** Cancellation path part one (architecture.md §21.3): save the checkpoint. */
|
|
134
|
+
private prepareCancelled;
|
|
135
|
+
/** Cancellation path part two: emit only after terminal lock ownership was proven. */
|
|
136
|
+
private emitCancelled;
|
|
137
|
+
private finishFailed;
|
|
138
|
+
/**
|
|
139
|
+
* Meldet eine Budgetverletzung der Queues. Eine Ueberschreitung des
|
|
140
|
+
* Speicherbudgets darf nie stillschweigend passieren (E4) — auch dann
|
|
141
|
+
* nicht, wenn sie unvermeidbar ist, weil ein einzelnes Element groesser
|
|
142
|
+
* als das ganze Budget ist.
|
|
143
|
+
*/
|
|
144
|
+
private reportOverBudget;
|
|
145
|
+
private runPipeline;
|
|
146
|
+
private saveCheckpoint;
|
|
147
|
+
}
|
|
148
|
+
//# sourceMappingURL=runner.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"runner.d.ts","sourceRoot":"","sources":["../src/runner.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;GAoBG;AACH,OAAO,KAAK,EACV,mBAAmB,EACnB,KAAK,EAEL,aAAa,EACb,QAAQ,EAER,YAAY,EACZ,UAAU,EACV,MAAM,EACP,MAAM,uBAAuB,CAAC;AAW/B,OAAO,EAA4C,KAAK,YAAY,EAAE,MAAM,oBAAoB,CAAC;AAEjG,OAAO,KAAK,EAAE,gBAAgB,EAAE,MAAM,aAAa,CAAC;AAEpD,OAAO,KAAK,EAAmC,cAAc,EAAiB,MAAM,aAAa,CAAC;AAGlG,4CAA4C;AAC5C,MAAM,MAAM,kBAAkB,GAAG,gBAAgB,CAAC;AAElD,MAAM,WAAW,uBAAuB;IACtC,wDAAwD;IACxD,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB,wDAAwD;IACxD,MAAM,CAAC,EAAE,MAAM,CAAC;CACjB;AAED,MAAM,WAAW,qBAAqB;IACpC,QAAQ,EAAE,kBAAkB,CAAC;IAC7B,MAAM,EAAE,cAAc,CAAC;IACvB,eAAe,EAAE,mBAAmB,CAAC;IACrC,KAAK,EAAE,KAAK,CAAC;IACb,MAAM,EAAE,MAAM,CAAC;IACf,MAAM,CAAC,EAAE,UAAU,CAAC;IACpB,6EAA6E;IAC7E,YAAY,CAAC,EAAE,YAAY,CAAC;IAC5B;;;;OAIG;IACH,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,2CAA2C;IAC3C,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,2EAA2E;IAC3E,YAAY,CAAC,EAAE,MAAM,CAAC;IACtB;;;;;OAKG;IACH,mBAAmB,CAAC,EAAE,CAAC,YAAY,EAAE,MAAM,KAAK,OAAO,CAAC,IAAI,CAAC,CAAC;IAC9D;;;;;;OAMG;IACH,oBAAoB,CAAC,EAAE,CACrB,YAAY,EAAE,MAAM,EACpB,OAAO,EAAE,WAAW,GAAG,WAAW,GAAG,QAAQ,EAC7C,yBAAyB,CAAC,EAAE,MAAM,KAC/B,OAAO,CAAC,IAAI,CAAC,CAAC;IACnB,+EAA+E;IAC/E,cAAc,CAAC,EAAE,SAAS,CAAC,MAAM,GAAG,QAAQ,CAAC,EAAE,CAAC;IAChD,wEAAwE;IACxE,cAAc,CAAC,EAAE,MAAM,CAAC;IACxB,eAAe,CAAC,EAAE,uBAAuB,CAAC;IAC1C;;;OAGG;IACH,iBAAiB,CAAC,EAAE,MAAM,CAAC;IAC3B,8EAA8E;IAC9E,MAAM,CAAC,EAAE,OAAO,CAAC;CAClB;AAED,gFAAgF;AAChF,MAAM,WAAW,WAAW;IAC1B,KAAK,EAAE,MAAM,CAAC;IACd,YAAY,EAAE,MAAM,GAAG,IAAI,CAAC;IAC5B,KAAK,EAAE,aAAa,CAAC;IACrB,YAAY,EAAE,MAAM,CAAC;IACrB,cAAc,EAAE,MAAM,CAAC;IACvB,iBAAiB,EAAE,MAAM,CAAC;IAC1B,WAAW,EAAE,MAAM,CAAC;IACpB,MAAM,EAAE;QAAE,QAAQ,EAAE,YAAY,CAAC;QAAC,MAAM,EAAE,YAAY,CAAA;KAAE,GAAG,IAAI,CAAC;CACjE;AA0HD,qBAAa,cAAc;IACzB,OAAO,CAAC,QAAQ,CAAC,OAAO,CAAwB;IAChD,OAAO,CAAC,QAAQ,CAAC,OAAO,CAAiC;IACzD,OAAO,CAAC,QAAQ,CAAC,KAAK,CAAS;IAC/B,OAAO,CAAC,YAAY,CAAuB;IAC3C,OAAO,CAAC,QAAQ,CAAC,cAAc,CAAqB;IACpD,OAAO,CAAC,oBAAoB,CAAgC;IAC5D,OAAO,CAAC,cAAc,CAA8B;IACpD,OAAO,CAAC,oBAAoB,CAAuC;IACnE,OAAO,CAAC,YAAY,CAAK;IACzB,OAAO,CAAC,MAAM,CAA+B;IAC7C,OAAO,CAAC,cAAc,CAAiE;IACvF,OAAO,CAAC,OAAO,CAAS;gBAEL,OAAO,EAAE,qBAAqB;IAMjD;;;;;;;;;;;OAWG;IACH,OAAO,CAAC,MAAM,CAAC,uBAAuB;IA2C/B,MAAM,IAAI,WAAW;IAiB5B;;;;;OAKG;IACI,GAAG,CAAC,OAAO,GAAE;QAAE,MAAM,CAAC,EAAE,WAAW,CAAA;KAAO,GAAG,aAAa,CAAC,QAAQ,CAAC;YA+C7D,KAAK;IAqQnB,+EAA+E;YACjE,gBAAgB;IAyB9B,sFAAsF;IACtF,OAAO,CAAC,aAAa;IASrB,OAAO,CAAC,YAAY;IAyBpB;;;;;OAKG;IACH,OAAO,CAAC,gBAAgB;YAmBV,WAAW;YAyJX,cAAc;CA2B7B"}
|