@untangle-code/jobs 0.0.0-stage → 0.1.1

package/src/stages.ts

@@ -0,0 +1,167 @@
+/**
+ * Narrow stage interfaces of the index pipeline (architecture.md §13.1).
+ *
+ * This package builds the pipeline MECHANICS — state machine, queues,
+ * backpressure, checkpoints, resume, cancellation, generation commit. The
+ * domain-specific stages (scanner, parser, chunker, embedder, stores) are
+ * INJECTED behind these interfaces; `@untangle-code/core` wires the real
+ * implementations later, tests wire deterministic fakes from
+ * `@untangle-code/testkit` ports.
+ *
+ * Every method takes trailing `PortCallOptions` and must respect
+ * `options.signal` (spec.md §22); the runner additionally only ever acts on
+ * results at safe batch boundaries (architecture.md §21.3).
+ */
+import type { ParseQuality, PortCallOptions } from "@untangle-code/domain";
+
+/** One file the discovery stage planned for (re)indexing. */
+export interface PlannedFileV1 {
+  fileId: string;
+  /** "/"-normalized repo-relative path. */
+  filePath: string;
+  /** Content hash — source of truth for change detection (spec.md §12.1). */
+  contentHash: string;
+}
+
+/** Result of the discovery stage (scanner + manifest comparison). */
+export interface DiscoveryResultV1 {
+  /** Files that need work in this run (changed or new). */
+  planned: PlannedFileV1[];
+  /**
+   * Every currently searchable source file, including unchanged files.
+   * This is kept separate from the resume checkpoint manifest: the latter
+   * proves confirmed writes, while this snapshot identifies the corpus the
+   * completed generation represents.
+   */
+  corpusManifestHashes: Record<string, string>;
+  totalFiles: number;
+  unchangedFiles: number;
+  ignoredFiles: number;
+}
+
+/** Ensures Qdrant/Ollama/collections are ready (spec.md §11.1 stage 2). */
+export interface InfrastructureStage {
+  ensureReady(options?: PortCallOptions): Promise<{
+    /** Effective endpoints with secrets redacted (spec.md §3.2). */
+    endpoints: Record<string, string>;
+  }>;
+}
+
+export interface DiscoveryStage {
+  discover(options?: PortCallOptions): Promise<DiscoveryResultV1>;
+}
+
+/** One prepared text ready for embedding, keyed by its future point id. */
+export interface PreparedTextV1 {
+  pointId: string;
+  text: string;
+  payload?: Record<string, unknown>;
+}
+
+/** Parse + chunk + metrics output for one file (stages 4–5 of spec.md §11.1). */
+export interface AnalyzedFileV1 {
+  file: PlannedFileV1;
+  parseQuality: ParseQuality;
+  chunkCount: number;
+  symbolCount: number;
+  preparedTexts: PreparedTextV1[];
+  /** Estimated in-memory size — feeds the queue byte budget (§13.2). */
+  estimatedBytes: number;
+}
+
+export interface AnalysisStage {
+  analyze(file: PlannedFileV1, options?: PortCallOptions): Promise<AnalyzedFileV1>;
+}
+
+/** One embedded point ready for upsert. */
+export interface EmbeddedPointV1 {
+  pointId: string;
+  fileId: string;
+  vector: number[];
+  payload?: Record<string, unknown>;
+}
+
+/** Result of embedding one file batch. */
+export interface EmbeddedBatchV1 {
+  files: PlannedFileV1[];
+  points: EmbeddedPointV1[];
+  cacheHits: number;
+  truncatedCount: number;
+  /** Estimated in-memory size — feeds the queue byte budget (§13.2). */
+  estimatedBytes: number;
+}
+
+export interface EmbeddingStage {
+  embedBatch(batch: AnalyzedFileV1[], options?: PortCallOptions): Promise<EmbeddedBatchV1>;
+}
+
+/**
+ * Confirmation of one CONFIRMED upsert. The runner persists a checkpoint
+ * only after receiving this (spec.md §10.5) — a storage stage must therefore
+ * resolve only once the store acknowledged the write.
+ */
+export interface UpsertConfirmationV1 {
+  collection: string;
+  pointCount: number;
+  serializedBytes: number;
+  /** Confirmed point ids per file id — recorded in the checkpoint (§13.3). */
+  storedPointIdsByFile: Record<string, string[]>;
+}
+
+export interface StorageStage {
+  upsertBatch(batch: EmbeddedBatchV1, options?: PortCallOptions): Promise<UpsertConfirmationV1>;
+}
+
+/** Result of building/patching one graph (stage `building_graphs`). */
+export interface GraphBuildResultV1 {
+  graph: "file" | "symbol";
+  nodeCount: number;
+  edgeCount: number;
+}
+
+export interface GraphStage {
+  buildGraphs(options?: PortCallOptions): Promise<GraphBuildResultV1[]>;
+}
+
+/** Optional context-artifact indexing (stage `indexing_context`). */
+export interface ContextStage {
+  indexContext(options?: PortCallOptions): Promise<void>;
+}
+
+/** Input the runner hands to the generation commit (architecture.md §13.4). */
+export interface GenerationCommitInputV1 {
+  jobId: string;
+  generationId: string;
+  workspaceId: string;
+  repoId: string;
+  refId: string;
+  effectiveProfileId: string;
+  plannedFileIds: string[];
+  confirmedFileIds: string[];
+  storedGraphs: string[];
+}
+
+export interface GenerationCommitResultV1 {
+  /** Must be "completed" for the job to finish as completed (§13.4). */
+  metadataStatus: string;
+}
+
+export interface GenerationCommitStage {
+  commit(
+    input: GenerationCommitInputV1,
+    options?: PortCallOptions,
+  ): Promise<GenerationCommitResultV1>;
+}
+
+/** The full injected stage set of one index job. */
+export interface IndexJobStages {
+  infrastructure: InfrastructureStage;
+  discovery: DiscoveryStage;
+  analysis: AnalysisStage;
+  embedding: EmbeddingStage;
+  storage: StorageStage;
+  graphs: GraphStage;
+  /** Optional — a workspace without context artifacts skips the stage's work. */
+  context?: ContextStage;
+  commit: GenerationCommitStage;
+}

package/src/state-machine.ts

@@ -0,0 +1,132 @@
+/**
+ * Full-index job state machine — exactly the stages and transitions of
+ * spec.md §11.1, including the error states `failed`, `cancel_requested`
+ * and `cancelled`. Illegal transitions are REJECTED with a typed error,
+ * never silently ignored.
+ */
+import { INDEX_JOB_STAGES, type IndexJobStage } from "@untangle-code/domain";
+import { indexIncompleteError } from "@untangle-code/observability";
+
+/**
+ * The linear pipeline chain of spec.md §11.1 (`queued` → … → `completed`).
+ * Derived from domain's `INDEX_JOB_STAGES` (the first twelve entries are the
+ * pipeline in order; the last three are the error states).
+ */
+export const JOB_PIPELINE_STAGES: readonly IndexJobStage[] = INDEX_JOB_STAGES.slice(
+  0,
+  INDEX_JOB_STAGES.indexOf("completed") + 1,
+);
+
+/** Terminal stages: no transition leaves them. */
+export const TERMINAL_JOB_STAGES: readonly IndexJobStage[] = ["completed", "failed", "cancelled"];
+
+/** True when `stage` is terminal (`completed`, `failed` or `cancelled`). */
+export function isTerminalJobStage(stage: IndexJobStage): boolean {
+  return TERMINAL_JOB_STAGES.includes(stage);
+}
+
+const PIPELINE_INDEX: ReadonlyMap<IndexJobStage, number> = new Map(
+  JOB_PIPELINE_STAGES.map((stage, index) => [stage, index]),
+);
+
+/** Index of `stage` in the pipeline chain, or null for the error states. */
+export function pipelineIndexOf(stage: IndexJobStage): number | null {
+  return PIPELINE_INDEX.get(stage) ?? null;
+}
+
+/**
+ * Pure transition rule of spec.md §11.1:
+ *
+ * - along the pipeline chain, only the single next stage is reachable;
+ * - `failed` and `cancel_requested` are reachable from every non-terminal
+ *   pipeline stage;
+ * - `cancel_requested` may only move to `cancelled` (or `failed`, when the
+ *   cancellation cleanup itself fails);
+ * - `completed`, `failed` and `cancelled` are terminal;
+ * - self-transitions are not transitions.
+ */
+export function canTransition(from: IndexJobStage, to: IndexJobStage): boolean {
+  if (from === to) {
+    return false;
+  }
+  if (isTerminalJobStage(from)) {
+    return false;
+  }
+  if (from === "cancel_requested") {
+    return to === "cancelled" || to === "failed";
+  }
+  // `from` is now a non-terminal pipeline stage.
+  if (to === "failed" || to === "cancel_requested") {
+    return true;
+  }
+  const fromIndex = pipelineIndexOf(from);
+  const toIndex = pipelineIndexOf(to);
+  return fromIndex !== null && toIndex !== null && toIndex === fromIndex + 1;
+}
+
+/**
+ * Mutable job stage holder enforcing spec.md §11.1. An illegal transition
+ * throws `UntangleError` with code `INDEX_INCOMPLETE` (the job would be left
+ * in an inconsistent state) — it never mutates and is never ignored.
+ */
+export class JobStateMachine {
+  #stage: IndexJobStage;
+
+  public constructor(initial: IndexJobStage = "queued") {
+    if (!INDEX_JOB_STAGES.includes(initial)) {
+      throw indexIncompleteError(`unknown job stage "${initial}"`, {
+        retryable: false,
+        operation: "job_stage_transition",
+        context: { stage: initial },
+      });
+    }
+    this.#stage = initial;
+  }
+
+  public get stage(): IndexJobStage {
+    return this.#stage;
+  }
+
+  /** Applies one legal transition or throws; never a silent no-op. */
+  public transition(to: IndexJobStage): void {
+    if (!canTransition(this.#stage, to)) {
+      throw indexIncompleteError(`illegal job stage transition "${this.#stage}" -> "${to}"`, {
+        retryable: false,
+        operation: "job_stage_transition",
+        context: { from: this.#stage, to },
+      });
+    }
+    this.#stage = to;
+  }
+
+  /**
+   * Walks the pipeline chain forward, one legal transition at a time, until
+   * `target` is reached. A `target` at or before the current stage is a
+   * no-op (used for stage high-water marks in the concurrent pipeline,
+   * see README "Entscheidungen"). Throws when the machine is in an error
+   * state or `target` is not a pipeline stage.
+   */
+  public advanceTo(target: IndexJobStage): void {
+    const targetIndex = pipelineIndexOf(target);
+    const currentIndex = pipelineIndexOf(this.#stage);
+    if (targetIndex === null || currentIndex === null) {
+      throw indexIncompleteError(
+        `cannot advance from "${this.#stage}" to "${target}": not a pipeline stage`,
+        {
+          retryable: false,
+          operation: "job_stage_transition",
+          context: { from: this.#stage, to: target },
+        },
+      );
+    }
+    if (targetIndex <= currentIndex) {
+      return;
+    }
+    for (let index = currentIndex + 1; index <= targetIndex; index += 1) {
+      const next = JOB_PIPELINE_STAGES[index];
+      if (next !== undefined) {
+        this.transition(next);
+      }
+    }
+  }
+}