@semiont/jobs 0.5.34 → 0.5.36

package/dist/index.d.ts

@@ -1,4 +1,4 @@
-import { JobId, UserId, ResourceId, EntityType, JobReferenceAnnotationResult, GenerationJobParams, JobHighlightAnnotationResult, JobAssessmentAnnotationResult, JobCommentAnnotationResult, TagSchema, JobTagAnnotationResult, Logger, EventBus, components, Annotation, SupportedMediaType, GatheredContext } from '@semiont/core';
+import { JobId, UserId, UnitCursor, ResourceId, EntityType, JobReferenceAnnotationResult, GenerationJobParams, JobHighlightAnnotationResult, JobAssessmentAnnotationResult, JobCommentAnnotationResult, TagSchema, JobTagAnnotationResult, Logger, EventBus, components, Annotation, SupportedMediaType, GatheredContext } from '@semiont/core';
 import { SemiontState } from '@semiont/core/node';
 import { InferenceClient } from '@semiont/inference';
 
@@ -47,6 +47,18 @@ interface JobMetadata {
      * duplicated.
      */
     completedUnits?: string[];
+    /**
+     * The finer grain `completedUnits` cannot express (CHUNK-GRAIN-RESUME P2):
+     * how far each UNFINISHED unit got, keyed by unit. Written per committed
+     * chunk, so a job that dies mid-unit resumes there rather than at the top —
+     * which for a one-unit job (every motivation job, and the 1958 document's
+     * single `Person` type) is the difference between resuming and restarting.
+     *
+     * A unit here is in progress, never complete; the two sets are disjoint by
+     * construction in `checkpointUnits`. Merged monotonically per unit, never
+     * unioned — see `mergeUnitCursors`.
+     */
+    unitCursors?: Record<string, UnitCursor>;
 }
 /**
  * Locale conventions for detection/generation params.
@@ -290,7 +302,7 @@ interface JobQueue {
      * `failureClass` of 'deterministic' goes straight to `failed` with any
      * budget remaining — a second identical attempt cannot succeed (P3).
      */
-    failJob(jobId: JobId, error: string, completedUnits?: string[], failureClass?: 'transient' | 'deterministic'): Promise<'retried' | 'failed' | null>;
+    failJob(jobId: JobId, error: string, completedUnits?: string[], failureClass?: 'transient' | 'deterministic', unitCursors?: Record<string, UnitCursor>): Promise<'retried' | 'failed' | null>;
     /**
      * Persist a running job's completed-unit checkpoint AT unit completion —
      * not only when a job fails (JOB-RESTART-SAFETY P2). `failJob` carries the
@@ -300,8 +312,25 @@ interface JobQueue {
      * file's metadata as each unit lands, unioned with any existing
      * checkpoint, so recovery resumes rather than restarts. Unthrottled (a
      * unit completion must never be dropped); a no-op for non-running jobs.
+     *
+     * `unitCursors` is the finer grain `completedUnits` cannot express
+     * (CHUNK-GRAIN-RESUME P2): how far an UNFINISHED unit got, so a job that
+     * dies mid-unit resumes there instead of at the top. It is written per
+     * committed chunk, not per unit.
+     *
+     * **The two merge differently, and the difference is the contract.**
+     * `completedUnits` is a set, so a union converges under concurrent snapshots
+     * — a set only grows. A cursor converges only if the merge is **monotone per
+     * unit**: a stale snapshot must never move one backward. `next` and `size`
+     * move TOGETHER as one observation; taking `size` from one snapshot and
+     * `next` from another would describe a chunk that never existed. A unit that
+     * reaches `completedUnits` drops its cursor, so "in progress with a cursor"
+     * and "complete" stay structurally exclusive rather than by convention.
+     *
+     * The rule belongs here rather than to any one driver because it is what
+     * makes a cursor safe to merge at all.
      */
-    checkpointUnits(jobId: JobId, completedUnits: string[]): Promise<void>;
+    checkpointUnits(jobId: JobId, completedUnits: string[], unitCursors?: Record<string, UnitCursor>): Promise<void>;
     /** Write progress into a running job's file (throttled, best-effort). */
     recordProgress(jobId: JobId, progress: Record<string, unknown>): Promise<void>;
     /**
@@ -385,7 +414,7 @@ declare class FsJobQueue implements JobQueue {
      * re-announced); after that it lands in `failed` with the error.
      * Returns null (and changes nothing) if the job isn't running.
      */
-    failJob(jobId: JobId, error: string, completedUnits?: string[], failureClass?: 'transient' | 'deterministic'): Promise<'retried' | 'failed' | null>;
+    failJob(jobId: JobId, error: string, completedUnits?: string[], failureClass?: 'transient' | 'deterministic', unitCursors?: Record<string, UnitCursor>): Promise<'retried' | 'failed' | null>;
     /**
      * Persist a running job's completed-unit checkpoint at unit completion
      * (JOB-RESTART-SAFETY P2). `failJob` writes this checkpoint on a clean
@@ -398,7 +427,7 @@ declare class FsJobQueue implements JobQueue {
      * non-running jobs. Written directly, like `recordProgress`, so it also
      * refreshes the mtime heartbeat.
      */
-    checkpointUnits(jobId: JobId, completedUnits: string[]): Promise<void>;
+    checkpointUnits(jobId: JobId, completedUnits: string[], unitCursors?: Record<string, UnitCursor>): Promise<void>;
     /**
      * Write progress into a running job's file. Throttled per job, and
      * a no-op for jobs that aren't running. Beyond surfacing live
@@ -521,15 +550,40 @@ type Motivation = Annotation['motivation'];
 interface ProcessorResult<R> {
     result: R;
 }
+/**
+ * Where one unit stands once the chunk just handed over is durable
+ * (CHUNK-GRAIN-RESUME P2).
+ *
+ * The unit is named HERE rather than in the detection layer, which knows about
+ * chunks and nothing about jobs: for `reference-annotation` a unit is an entity
+ * type, for `tag-annotation` a category — both loop over several — and for the
+ * other three the job runs exactly one unit, its own motivation.
+ */
+interface UnitCheckpoint {
+    unit: string;
+    cursor: UnitCursor;
+}
 declare function processHighlightJob(content: string, inferenceClient: InferenceClient, params: HighlightDetectionParams, buildAnnotation: BuildAnnotation, onProgress: OnProgress, 
 /** This chunk's novel annotations, awaited: the durability write. */
-onChunkComplete: (annotations: Annotation[]) => Promise<void>): Promise<ProcessorResult<JobHighlightAnnotationResult>>;
+onChunkComplete: (annotations: Annotation[], checkpoint: UnitCheckpoint) => Promise<void>, 
+/** Where earlier attempts left each unit (CHUNK-GRAIN-RESUME P3), keyed the
+ * same way the checkpoint is. A unit absent here starts at the top, which is
+ * every unit of a first attempt. */
+resumeCursors?: Record<string, UnitCursor>): Promise<ProcessorResult<JobHighlightAnnotationResult>>;
 declare function processCommentJob(content: string, inferenceClient: InferenceClient, params: CommentDetectionParams, buildAnnotation: BuildAnnotation, onProgress: OnProgress, 
 /** This chunk's novel annotations, awaited: the durability write. */
-onChunkComplete: (annotations: Annotation[]) => Promise<void>): Promise<ProcessorResult<JobCommentAnnotationResult>>;
+onChunkComplete: (annotations: Annotation[], checkpoint: UnitCheckpoint) => Promise<void>, 
+/** Where earlier attempts left each unit (CHUNK-GRAIN-RESUME P3), keyed the
+ * same way the checkpoint is. A unit absent here starts at the top, which is
+ * every unit of a first attempt. */
+resumeCursors?: Record<string, UnitCursor>): Promise<ProcessorResult<JobCommentAnnotationResult>>;
 declare function processAssessmentJob(content: string, inferenceClient: InferenceClient, params: AssessmentDetectionParams, buildAnnotation: BuildAnnotation, onProgress: OnProgress, 
 /** This chunk's novel annotations, awaited: the durability write. */
-onChunkComplete: (annotations: Annotation[]) => Promise<void>): Promise<ProcessorResult<JobAssessmentAnnotationResult>>;
+onChunkComplete: (annotations: Annotation[], checkpoint: UnitCheckpoint) => Promise<void>, 
+/** Where earlier attempts left each unit (CHUNK-GRAIN-RESUME P3), keyed the
+ * same way the checkpoint is. A unit absent here starts at the top, which is
+ * every unit of a first attempt. */
+resumeCursors?: Record<string, UnitCursor>): Promise<ProcessorResult<JobAssessmentAnnotationResult>>;
 /**
  * Reference detection commits per UNIT — one entity type — through
  * `onUnitComplete` (ABANDONED-INFERENCE P2, checkpointed resume): the
@@ -548,12 +602,20 @@ declare function processReferenceJob(content: string, inferenceClient: Inference
  */
 onUnitComplete: (entityType: string) => Promise<void>, signal?: AbortSignal, 
 /** This chunk's novel annotations, awaited: the durability write. */
-onChunkComplete?: (annotations: Annotation[]) => Promise<void>): Promise<{
+onChunkComplete?: (annotations: Annotation[], checkpoint: UnitCheckpoint) => Promise<void>, 
+/** Where earlier attempts left each entity-type unit (CHUNK-GRAIN-RESUME P3).
+ * A unit absent here starts at the top; units already COMPLETE never reach
+ * this function at all, the caller having filtered them out. */
+resumeCursors?: Record<string, UnitCursor>): Promise<{
     result: JobReferenceAnnotationResult;
 }>;
 declare function processTagJob(content: string, inferenceClient: InferenceClient, params: TagDetectionParams, buildAnnotation: BuildAnnotation, onProgress: OnProgress, 
 /** This chunk's novel annotations, awaited: the durability write. */
-onChunkComplete: (annotations: Annotation[]) => Promise<void>): Promise<ProcessorResult<JobTagAnnotationResult>>;
+onChunkComplete: (annotations: Annotation[], checkpoint: UnitCheckpoint) => Promise<void>, 
+/** Where earlier attempts left each CATEGORY (CHUNK-GRAIN-RESUME P3) — a tag
+ * job's units are its categories, not its motivation: each walks the whole
+ * document, so one shared cursor would skip text for all but one of them. */
+resumeCursors?: Record<string, UnitCursor>): Promise<ProcessorResult<JobTagAnnotationResult>>;
 declare function processGenerationJob(inferenceClient: InferenceClient, params: GenerationJobParams, onProgress: OnProgress, logger: Logger): Promise<{
     content: Uint8Array;
     title: string;
@@ -562,6 +624,44 @@ declare function processGenerationJob(inferenceClient: InferenceClient, params:
     result: GenerationResult;
 }>;
 
+/**
+ * Detection budget derivation — pure window arithmetic over the inference
+ * provider's published limits. No hand-tuned chunk constants, no density or
+ * yield modeling: document content never enters this arithmetic. The guard
+ * for the pathological tail (a chunk whose annotation JSON still overflows
+ * the output budget) is `assertNotTruncated` below — invoked per chunk by
+ * the callers on every response, not a prediction here.
+ *
+ * Provider shapes (see `@semiont/inference` interface.ts):
+ * - Shared window (Ollama): the provider publishes
+ *   `maxOutputTokens === contextTokens` — prompt and response share one
+ *   `num_ctx`. What remains after the prompt scaffold is split
+ *   input:output = 1:2 (annotation JSON echoes each span plus a fixed
+ *   key/context envelope, so output needs the larger share). The 1:2 ratio
+ *   is the plan's one allocation policy — doc-independent, tuned only on
+ *   live evidence.
+ * - Separate ceilings (Anthropic): output takes its full ceiling (duration-
+ *   capped below), and input follows the same 1:2 allocation — never more
+ *   than half the output budget. Input does NOT get "the rest of the
+ *   window": measured 2026-09-02, a window-sized chunk of entity-dense text
+ *   demands more output than any budget holds, so the model grinds toward
+ *   max_tokens for minutes (killed at the call bound as a "stall") or
+ *   collapses to the degenerate []. Large documents chunk; that is the fix,
+ *   not a cost.
+ */
+
+/** One chunk handed out by `runAdaptiveChunks`, with the cursor either side of
+ * it. `at`/`next` over `totalChars` is exact progress — and, once
+ * CHUNK-GRAIN-RESUME lands, the checkpoint identity a variable boundary forces
+ * (an ordinal cannot name a chunk whose size is decided while the job runs). */
+/**
+ * The half of a `UnitCursor` the DETECTION layer can honestly report: where the
+ * walk stands and how it is cutting. The tallies belong to the processor, which
+ * owns unit identity and unit counts — this loop counts chunks, not annotations,
+ * and a zero it invented would be indistinguishable from a real one.
+ */
+type ChunkCursor = Pick<UnitCursor, 'next' | 'size'>;
+
 /**
  * Response parsers for annotation detection motivations
  *
@@ -638,9 +738,12 @@ declare class AnnotationDetection {
      * (source-resource locale). See `types.ts` "Locale conventions" for the
      * full discussion.
      */
-    static detectComments(content: string, client: InferenceClient, instructions?: string, tone?: string, density?: number, language?: string, sourceLanguage?: string, onActivity?: (completedChunks: number, totalChunks: number) => void, 
+    static detectComments(content: string, client: InferenceClient, instructions?: string, tone?: string, density?: number, language?: string, sourceLanguage?: string, onActivity?: (consumedChars: number, totalChars: number) => void, 
     /** This chunk's matches, as the chunk completes. */
-    onChunkResults?: (matches: CommentMatch[]) => Promise<void>): Promise<CommentMatch[]>;
+    /** Where an earlier attempt left this unit (CHUNK-GRAIN-RESUME P3). */
+    resume?: UnitCursor, 
+    /** This chunk's matches, as the chunk completes. Kept LAST. */
+    onChunkResults?: (matches: CommentMatch[], cursor: ChunkCursor) => Promise<void>): Promise<CommentMatch[]>;
     /**
      * Detect highlights in content.
      *
@@ -648,9 +751,12 @@ declare class AnnotationDetection {
      * applies, used in the prompt so the LLM analyzes non-English source
      * correctly.
      */
-    static detectHighlights(content: string, client: InferenceClient, instructions?: string, density?: number, sourceLanguage?: string, onActivity?: (completedChunks: number, totalChunks: number) => void, 
+    static detectHighlights(content: string, client: InferenceClient, instructions?: string, density?: number, sourceLanguage?: string, onActivity?: (consumedChars: number, totalChars: number) => void, 
     /** This chunk's matches, as the chunk completes. */
-    onChunkResults?: (matches: HighlightMatch[]) => Promise<void>): Promise<HighlightMatch[]>;
+    /** Where an earlier attempt left this unit (CHUNK-GRAIN-RESUME P3). */
+    resume?: UnitCursor, 
+    /** This chunk's matches, as the chunk completes. Kept LAST. */
+    onChunkResults?: (matches: HighlightMatch[], cursor: ChunkCursor) => Promise<void>): Promise<HighlightMatch[]>;
     /**
      * Detect assessments in content.
      *
@@ -658,9 +764,12 @@ declare class AnnotationDetection {
      * (annotation body locale). `sourceLanguage` is the locale of the content
      * being analyzed (source-resource locale).
      */
-    static detectAssessments(content: string, client: InferenceClient, instructions?: string, tone?: string, density?: number, language?: string, sourceLanguage?: string, onActivity?: (completedChunks: number, totalChunks: number) => void, 
+    static detectAssessments(content: string, client: InferenceClient, instructions?: string, tone?: string, density?: number, language?: string, sourceLanguage?: string, onActivity?: (consumedChars: number, totalChars: number) => void, 
     /** This chunk's matches, as the chunk completes. */
-    onChunkResults?: (matches: AssessmentMatch[]) => Promise<void>): Promise<AssessmentMatch[]>;
+    /** Where an earlier attempt left this unit (CHUNK-GRAIN-RESUME P3). */
+    resume?: UnitCursor, 
+    /** This chunk's matches, as the chunk completes. Kept LAST. */
+    onChunkResults?: (matches: AssessmentMatch[], cursor: ChunkCursor) => Promise<void>): Promise<AssessmentMatch[]>;
     /**
      * Detect tags in content for a specific category.
      *
@@ -673,13 +782,16 @@ declare class AnnotationDetection {
      * identifiers, not LLM-generated text — so it's consumed at the body-stamp
      * site, not here.
      */
-    static detectTags(content: string, client: InferenceClient, schema: TagSchema, category: string, sourceLanguage?: string, onActivity?: (completedChunks: number, totalChunks: number) => void, 
+    static detectTags(content: string, client: InferenceClient, schema: TagSchema, category: string, sourceLanguage?: string, onActivity?: (consumedChars: number, totalChars: number) => void, 
     /**
      * This chunk's matches, ANCHORED before they leave: `parse` here yields
      * raw tags, so this path runs `validateTagOffsets` per chunk — a per-item
      * anchor against the full document, so partitioning changes nothing.
      */
-    onChunkResults?: (matches: TagMatch[]) => Promise<void>): Promise<TagMatch[]>;
+    /** Where an earlier attempt left this unit (CHUNK-GRAIN-RESUME P3). */
+    resume?: UnitCursor, 
+    /** This chunk's matches, as the chunk completes. Kept LAST. */
+    onChunkResults?: (matches: TagMatch[], cursor: ChunkCursor) => Promise<void>): Promise<TagMatch[]>;
 }
 
 /**