@untangle-code/jobs 0.0.0-stage → 0.1.1

package/src/events.ts

@@ -0,0 +1,71 @@
+/**
+ * Single-consumer async event channel — the transport that makes the typed
+ * job events of architecture.md §2.2 consumable as an `AsyncIterable`
+ * (spec.md §22: `index()` returns `AsyncIterable<JobEvent>`).
+ *
+ * The producer side (`emit`) never blocks: job events are small, bounded in
+ * number per batch, and the pipeline's memory pressure is controlled by the
+ * bounded work queues, not by the event stream. `fail()` exists for
+ * catastrophic driver bugs only — regular job failures are emitted as
+ * `job_failed` events and end the stream via `close()`.
+ */
+
+export class AsyncEventChannel<T> implements AsyncIterable<T> {
+  private readonly buffer: T[] = [];
+  private closed = false;
+  private failure: { error: unknown } | null = null;
+  private wakers: Array<() => void> = [];
+
+  /** Enqueues one event. Throws when the channel is already closed/failed. */
+  public emit(event: T): void {
+    if (this.closed) {
+      throw new Error("AsyncEventChannel.emit() called after close()/fail()");
+    }
+    this.buffer.push(event);
+    this.notify();
+  }
+
+  /** Ends the stream after all buffered events were consumed. Idempotent. */
+  public close(): void {
+    this.closed = true;
+    this.notify();
+  }
+
+  /**
+   * Ends the stream with a rejection after all buffered events were
+   * consumed. Idempotent; a later `fail()` does not override an earlier one.
+   */
+  public fail(error: unknown): void {
+    if (this.failure === null) {
+      this.failure = { error };
+    }
+    this.closed = true;
+    this.notify();
+  }
+
+  private notify(): void {
+    const wakers = this.wakers;
+    this.wakers = [];
+    for (const wake of wakers) {
+      wake();
+    }
+  }
+
+  public async *[Symbol.asyncIterator](): AsyncIterator<T> {
+    while (true) {
+      if (this.buffer.length > 0) {
+        // `length > 0` guarantees the shift result is a T, even for a T that
+        // could itself include `undefined`.
+        yield this.buffer.shift() as T;
+        continue;
+      }
+      if (this.failure !== null) {
+        throw this.failure.error;
+      }
+      if (this.closed) {
+        return;
+      }
+      await new Promise<void>((resolve) => this.wakers.push(resolve));
+    }
+  }
+}

package/src/index.ts

@@ -0,0 +1,37 @@
+/**
+ * @untangle-code/jobs — the job core of the Untangle indexer
+ * (spec.md §11, §12.2; architecture.md §2.2, §6.14, §13, §21.3–§21.4).
+ *
+ * Pipeline MECHANICS only: the exact spec.md §11.1 state machine, typed job
+ * events as `AsyncIterable`, a checkpoint store (`IndexCheckpointV1`,
+ * checkpoints only after confirmed upserts), resume planning, generation
+ * commit per §13.4, bounded queues with backpressure and a visible memory
+ * budget, and `AbortSignal` cancellation at safe batch boundaries. The
+ * domain-specific stages (scanner, parser, chunker, embedder, stores) are
+ * injected behind narrow interfaces. This module is the ONLY public export
+ * of the package.
+ */
+
+// Abort helpers (spec.md §22).
+export * from "./abort.js";
+
+// State machine (spec.md §11.1).
+export * from "./state-machine.js";
+
+// Async event channel (architecture.md §2.2).
+export * from "./events.js";
+
+// Bounded queues with backpressure (architecture.md §13.2).
+export * from "./bounded-queue.js";
+
+// Injected stage interfaces (architecture.md §13.1).
+export * from "./stages.js";
+
+// Checkpoint store (architecture.md §13.3; spec.md §10.5).
+export * from "./checkpoint-store.js";
+
+// Resume planning (spec.md §11.2).
+export * from "./resume.js";
+
+// Index job runner (spec.md §11; architecture.md §13.4, §21.3).
+export * from "./runner.js";

package/src/resume.ts

@@ -0,0 +1,106 @@
+/**
+ * Resume planning (architecture.md §6.14): reconstruct the remaining work
+ * plan from the last CONFIRMED checkpoint, so a restarted job continues
+ * exactly where confirmed work ended (spec.md §11.2) and never repeats an
+ * already-confirmed file.
+ */
+import type { IndexCheckpointV1 } from "@untangle-code/domain";
+
+import type { PlannedFileV1 } from "./stages.js";
+
+/** The identity a checkpoint must match to be resumable for this job. */
+export interface ResumeIdentityV1 {
+  workspaceId: string;
+  repoId: string;
+  refId: string;
+  effectiveProfileId: string;
+}
+
+/**
+ * True when `checkpoint` belongs to exactly this workspace/repo/ref/profile.
+ * A profile mismatch means an incompatible embedding profile — such a
+ * checkpoint MUST NOT be resumed; the job needs its own new generation
+ * (spec.md §11.2).
+ */
+export function checkpointMatches(
+  checkpoint: IndexCheckpointV1,
+  identity: ResumeIdentityV1,
+): boolean {
+  return (
+    checkpoint.workspaceId === identity.workspaceId &&
+    checkpoint.repoId === identity.repoId &&
+    checkpoint.refId === identity.refId &&
+    checkpoint.effectiveProfileId === identity.effectiveProfileId
+  );
+}
+
+export interface ResumePlanV1 {
+  /** True when a matching checkpoint was adopted. */
+  resumed: boolean;
+  /** Generation to continue (from the checkpoint), or null for a fresh run. */
+  generationId: string | null;
+  /** Files that still need the full pipeline. */
+  remaining: PlannedFileV1[];
+  /** Confirmed state carried over from the checkpoint (planned ∩ unchanged). */
+  completedFiles: string[];
+  storedPointIdsByFile: Record<string, string[]>;
+  manifestHashes: Record<string, string>;
+  /** How many planned files were skipped because they are already confirmed. */
+  skippedCompleted: number;
+}
+
+/**
+ * Builds the remaining work plan. A planned file is skipped only when the
+ * checkpoint confirms it AND its content hash is unchanged — the content
+ * hash is the source of truth (spec.md §12.1). Confirmed files whose hash
+ * changed (or that are no longer planned) are dropped from the carried
+ * state so they are re-confirmed by this run, never counted twice.
+ */
+export function planResume(
+  planned: readonly PlannedFileV1[],
+  checkpoint: IndexCheckpointV1 | null,
+  identity: ResumeIdentityV1,
+): ResumePlanV1 {
+  if (checkpoint === null || !checkpointMatches(checkpoint, identity)) {
+    return {
+      resumed: false,
+      generationId: null,
+      remaining: [...planned],
+      completedFiles: [],
+      storedPointIdsByFile: {},
+      manifestHashes: {},
+      skippedCompleted: 0,
+    };
+  }
+
+  const confirmed = new Set(checkpoint.completedFiles);
+  const remaining: PlannedFileV1[] = [];
+  const completedFiles: string[] = [];
+  const storedPointIdsByFile: Record<string, string[]> = {};
+  const manifestHashes: Record<string, string> = {};
+
+  for (const file of planned) {
+    const isConfirmedUnchanged =
+      confirmed.has(file.fileId) && checkpoint.manifestHashes[file.fileId] === file.contentHash;
+    if (!isConfirmedUnchanged) {
+      remaining.push(file);
+      continue;
+    }
+    completedFiles.push(file.fileId);
+    manifestHashes[file.fileId] = file.contentHash;
+    const pointIds = checkpoint.storedPointIdsByFile[file.fileId];
+    if (pointIds !== undefined) {
+      storedPointIdsByFile[file.fileId] = [...pointIds];
+    }
+  }
+
+  return {
+    resumed: true,
+    generationId: checkpoint.generationId,
+    remaining,
+    completedFiles,
+    storedPointIdsByFile,
+    manifestHashes,
+    skippedCompleted: completedFiles.length,
+  };
+}