@oscharko-dev/keiko-local-knowledge 0.2.0

package/dist/indexing/job-persist.d.ts

@@ -0,0 +1,46 @@
+import type { IndexingJobError, IndexingJobRecord, IndexingJobStatus, KnowledgeCapsuleId, KnowledgeSourceId } from "@oscharko-dev/keiko-contracts";
+import type { DatabaseSync } from "node:sqlite";
+export interface IndexingJobRow {
+    readonly id: string;
+    readonly capsule_id: string;
+    readonly source_ids_json: string;
+    readonly started_at: number;
+    readonly finished_at: number | null;
+    readonly status: string;
+    readonly total_documents: number;
+    readonly processed_documents: number;
+    readonly failed_documents: number;
+    readonly skipped_documents: number;
+    readonly last_error_code: string | null;
+    readonly last_error_message: string | null;
+    readonly resume_token: string | null;
+    readonly cancellation_requested: number;
+}
+export interface InsertJobInput {
+    readonly id: string;
+    readonly capsuleId: KnowledgeCapsuleId;
+    readonly sourceIds: readonly KnowledgeSourceId[];
+    readonly startedAt: number;
+}
+export declare function insertJobRow(db: DatabaseSync, input: InsertJobInput): void;
+export interface JobCounters {
+    readonly total: number;
+    readonly processed: number;
+    readonly failed: number;
+    readonly skipped: number;
+    readonly resumeToken: string | null;
+}
+export declare function updateJobCounters(db: DatabaseSync, id: string, counters: JobCounters): void;
+export interface FinalizeJobInput {
+    readonly id: string;
+    readonly status: IndexingJobStatus;
+    readonly finishedAt: number;
+    readonly counters: JobCounters;
+    readonly lastError?: IndexingJobError;
+}
+export declare function finalizeJobRow(db: DatabaseSync, input: FinalizeJobInput): void;
+export declare function selectRunningJobByCapsule(db: DatabaseSync, capsuleId: KnowledgeCapsuleId): IndexingJobRow | undefined;
+export declare function selectJobById(db: DatabaseSync, id: string): IndexingJobRow | undefined;
+export declare function isJobCancellationRequested(db: DatabaseSync, id: string): boolean;
+export declare function rowToIndexingJobRecord(row: IndexingJobRow): IndexingJobRecord;
+//# sourceMappingURL=job-persist.d.ts.map

package/dist/indexing/job-persist.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"job-persist.d.ts","sourceRoot":"","sources":["../../src/indexing/job-persist.ts"],"names":[],"mappings":"AAaA,OAAO,KAAK,EACV,gBAAgB,EAChB,iBAAiB,EACjB,iBAAiB,EACjB,kBAAkB,EAClB,iBAAiB,EAClB,MAAM,+BAA+B,CAAC;AACvC,OAAO,KAAK,EAAE,YAAY,EAAE,MAAM,aAAa,CAAC;AAyDhD,MAAM,WAAW,cAAc;IAC7B,QAAQ,CAAC,EAAE,EAAE,MAAM,CAAC;IACpB,QAAQ,CAAC,UAAU,EAAE,MAAM,CAAC;IAC5B,QAAQ,CAAC,eAAe,EAAE,MAAM,CAAC;IACjC,QAAQ,CAAC,UAAU,EAAE,MAAM,CAAC;IAC5B,QAAQ,CAAC,WAAW,EAAE,MAAM,GAAG,IAAI,CAAC;IACpC,QAAQ,CAAC,MAAM,EAAE,MAAM,CAAC;IACxB,QAAQ,CAAC,eAAe,EAAE,MAAM,CAAC;IACjC,QAAQ,CAAC,mBAAmB,EAAE,MAAM,CAAC;IACrC,QAAQ,CAAC,gBAAgB,EAAE,MAAM,CAAC;IAClC,QAAQ,CAAC,iBAAiB,EAAE,MAAM,CAAC;IACnC,QAAQ,CAAC,eAAe,EAAE,MAAM,GAAG,IAAI,CAAC;IACxC,QAAQ,CAAC,kBAAkB,EAAE,MAAM,GAAG,IAAI,CAAC;IAC3C,QAAQ,CAAC,YAAY,EAAE,MAAM,GAAG,IAAI,CAAC;IACrC,QAAQ,CAAC,sBAAsB,EAAE,MAAM,CAAC;CACzC;AAED,MAAM,WAAW,cAAc;IAC7B,QAAQ,CAAC,EAAE,EAAE,MAAM,CAAC;IACpB,QAAQ,CAAC,SAAS,EAAE,kBAAkB,CAAC;IACvC,QAAQ,CAAC,SAAS,EAAE,SAAS,iBAAiB,EAAE,CAAC;IACjD,QAAQ,CAAC,SAAS,EAAE,MAAM,CAAC;CAC5B;AAED,wBAAgB,YAAY,CAAC,EAAE,EAAE,YAAY,EAAE,KAAK,EAAE,cAAc,GAAG,IAAI,CAQ1E;AAED,MAAM,WAAW,WAAW;IAC1B,QAAQ,CAAC,KAAK,EAAE,MAAM,CAAC;IACvB,QAAQ,CAAC,SAAS,EAAE,MAAM,CAAC;IAC3B,QAAQ,CAAC,MAAM,EAAE,MAAM,CAAC;IACxB,QAAQ,CAAC,OAAO,EAAE,MAAM,CAAC;IACzB,QAAQ,CAAC,WAAW,EAAE,MAAM,GAAG,IAAI,CAAC;CACrC;AAED,wBAAgB,iBAAiB,CAAC,EAAE,EAAE,YAAY,EAAE,EAAE,EAAE,MAAM,EAAE,QAAQ,EAAE,WAAW,GAAG,IAAI,CAS3F;AAED,MAAM,WAAW,gBAAgB;IAC/B,QAAQ,CAAC,EAAE,EAAE,MAAM,CAAC;IACpB,QAAQ,CAAC,MAAM,EAAE,iBAAiB,CAAC;IACnC,QAAQ,CAAC,UAAU,EAAE,MAAM,CAAC;IAC5B,QAAQ,CAAC,QAAQ,EAAE,WAAW,CAAC;IAC/B,QAAQ,CAAC,SAAS,CAAC,EAAE,gBAAgB,CAAC;CACvC;AAED,wBAAgB,cAAc,CAAC,EAAE,EAAE,YAAY,EAAE,KAAK,EAAE,gBAAgB,GAAG,IAAI,CAa9E;AAED,wBAAgB,yBAAyB,CACvC,EAAE,EAAE,YAAY,EAChB,SAAS,EAAE,kBAAkB,GAC5B,cAAc,GAAG,SAAS,CAG5B;AAED,wBAAgB,aAAa,CAAC,EAAE,EAAE,YAAY,EAAE,EAAE,EAAE,MAAM,GAAG,cAAc,GAAG,SAAS,CAGtF;AAED,wBAAgB,0BAA0B,CAAC,EAAE,EAAE,YAAY,EAAE,EAAE,EAAE,MAAM,GAAG,OAAO,CAKhF;AA4CD,wBAAgB,sBAAsB,CAAC,GAAG,EAAE,cAAc,GAAG,iBAAiB,CAa7E"}

package/dist/indexing/job-persist.js

@@ -0,0 +1,157 @@
+// Prepared-statement helpers for the `indexing_jobs` table (Epic #189, Issue #196). The
+// orchestrator writes one row per run and updates it through the lifecycle:
+//
+//   started → progress (per document) → succeeded | failed | cancelled
+//
+// The schema reserves `cancellation_requested` (INTEGER 0/1) for a future external-cancel
+// surface; this layer writes it but does not poll it (the orchestrator drives cancellation
+// purely via AbortSignal).
+//
+// `resume_token` is set to the lexicographically-greatest `chunk_id` embedded so far so a
+// subsequent resume can continue past it. The orchestrator's incremental skip still scopes
+// to documents — the token is purely diagnostic for now and is consumed by #198 UI later.
+const INSERT_JOB_SQL = [
+    "INSERT INTO indexing_jobs (",
+    "  id, capsule_id, source_ids_json, started_at, finished_at, status,",
+    "  total_documents, processed_documents, failed_documents, skipped_documents,",
+    "  last_error_code, last_error_message, resume_token, cancellation_requested",
+    ") VALUES (",
+    "  :id, :capsule_id, :source_ids_json, :started_at, NULL, :status,",
+    "  0, 0, 0, 0, NULL, NULL, NULL, 0",
+    ")",
+].join(" ");
+const UPDATE_COUNTERS_SQL = [
+    "UPDATE indexing_jobs SET",
+    "  total_documents = :total,",
+    "  processed_documents = :processed,",
+    "  failed_documents = :failed,",
+    "  skipped_documents = :skipped,",
+    "  resume_token = :resume",
+    "WHERE id = :id",
+].join(" ");
+const FINALIZE_JOB_SQL = [
+    "UPDATE indexing_jobs SET",
+    "  status = :status,",
+    "  finished_at = :finished_at,",
+    "  total_documents = :total,",
+    "  processed_documents = :processed,",
+    "  failed_documents = :failed,",
+    "  skipped_documents = :skipped,",
+    "  last_error_code = :error_code,",
+    "  last_error_message = :error_message,",
+    "  resume_token = :resume",
+    "WHERE id = :id",
+].join(" ");
+const SELECT_RUNNING_BY_CAPSULE_SQL = [
+    "SELECT id, capsule_id, source_ids_json, started_at, finished_at, status,",
+    "  total_documents, processed_documents, failed_documents, skipped_documents,",
+    "  last_error_code, last_error_message, resume_token, cancellation_requested",
+    "FROM indexing_jobs",
+    "WHERE capsule_id = :c AND status = 'running'",
+    // started_at DESC so the most recent abandoned run wins. Ties broken by id ASC for
+    // deterministic resume behaviour across clocks that issue duplicate ms timestamps.
+    "ORDER BY started_at DESC, id ASC",
+    "LIMIT 1",
+].join(" ");
+const SELECT_BY_ID_SQL = [
+    "SELECT id, capsule_id, source_ids_json, started_at, finished_at, status,",
+    "  total_documents, processed_documents, failed_documents, skipped_documents,",
+    "  last_error_code, last_error_message, resume_token, cancellation_requested",
+    "FROM indexing_jobs",
+    "WHERE id = :id",
+].join(" ");
+export function insertJobRow(db, input) {
+    db.prepare(INSERT_JOB_SQL).run({
+        id: input.id,
+        capsule_id: String(input.capsuleId),
+        source_ids_json: JSON.stringify(input.sourceIds.map((s) => String(s))),
+        started_at: input.startedAt,
+        status: "running",
+    });
+}
+export function updateJobCounters(db, id, counters) {
+    db.prepare(UPDATE_COUNTERS_SQL).run({
+        id,
+        total: counters.total,
+        processed: counters.processed,
+        failed: counters.failed,
+        skipped: counters.skipped,
+        resume: counters.resumeToken,
+    });
+}
+export function finalizeJobRow(db, input) {
+    db.prepare(FINALIZE_JOB_SQL).run({
+        id: input.id,
+        status: input.status,
+        finished_at: input.finishedAt,
+        total: input.counters.total,
+        processed: input.counters.processed,
+        failed: input.counters.failed,
+        skipped: input.counters.skipped,
+        error_code: input.lastError?.code ?? null,
+        error_message: input.lastError?.message ?? null,
+        resume: input.counters.resumeToken,
+    });
+}
+export function selectRunningJobByCapsule(db, capsuleId) {
+    const row = db.prepare(SELECT_RUNNING_BY_CAPSULE_SQL).get({ c: String(capsuleId) });
+    return row === undefined ? undefined : row;
+}
+export function selectJobById(db, id) {
+    const row = db.prepare(SELECT_BY_ID_SQL).get({ id });
+    return row === undefined ? undefined : row;
+}
+export function isJobCancellationRequested(db, id) {
+    const row = db
+        .prepare("SELECT cancellation_requested FROM indexing_jobs WHERE id = :id")
+        .get({ id });
+    return row?.cancellation_requested === 1;
+}
+// ─── Row → IndexingJobRecord ──────────────────────────────────────────────────
+function parseSourceIds(json) {
+    const parsed = JSON.parse(json);
+    if (!Array.isArray(parsed))
+        return [];
+    return parsed
+        .filter((entry) => typeof entry === "string")
+        .map((entry) => entry);
+}
+const VALID_STATUSES = new Set([
+    "queued",
+    "running",
+    "succeeded",
+    "failed",
+    "cancelled",
+]);
+function parseStatus(raw) {
+    // The schema does not CHECK status, so a corrupt row could carry an unexpected string.
+    // Default to "failed" so consumers do not silently accept a row that violates the closed
+    // union — the row remains visible (operators can recover it manually).
+    return VALID_STATUSES.has(raw) ? raw : "failed";
+}
+function buildLastError(row) {
+    if (row.last_error_code === null || row.last_error_message === null)
+        return undefined;
+    return { code: row.last_error_code, message: row.last_error_message };
+}
+function optionalJobFields(row) {
+    const lastError = buildLastError(row);
+    return {
+        ...(row.finished_at !== null ? { finishedAt: row.finished_at } : {}),
+        ...(lastError !== undefined ? { lastError } : {}),
+    };
+}
+export function rowToIndexingJobRecord(row) {
+    return {
+        id: row.id,
+        capsuleId: row.capsule_id,
+        sourceIds: parseSourceIds(row.source_ids_json),
+        startedAt: row.started_at,
+        status: parseStatus(row.status),
+        totalDocuments: row.total_documents,
+        processedDocuments: row.processed_documents,
+        failedDocuments: row.failed_documents,
+        skippedDocuments: row.skipped_documents,
+        ...optionalJobFields(row),
+    };
+}

package/dist/indexing/job-resume.d.ts

@@ -0,0 +1,4 @@
+import type { IndexingJobRecord, KnowledgeCapsuleId } from "@oscharko-dev/keiko-contracts";
+import type { KnowledgeStore } from "../store.js";
+export declare function findResumableJob(store: KnowledgeStore, capsuleId: KnowledgeCapsuleId): IndexingJobRecord | undefined;
+//# sourceMappingURL=job-resume.d.ts.map

package/dist/indexing/job-resume.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"job-resume.d.ts","sourceRoot":"","sources":["../../src/indexing/job-resume.ts"],"names":[],"mappings":"AAUA,OAAO,KAAK,EAAE,iBAAiB,EAAE,kBAAkB,EAAE,MAAM,+BAA+B,CAAC;AAG3F,OAAO,KAAK,EAAE,cAAc,EAAE,MAAM,aAAa,CAAC;AAElD,wBAAgB,gBAAgB,CAC9B,KAAK,EAAE,cAAc,EACrB,SAAS,EAAE,kBAAkB,GAC5B,iBAAiB,GAAG,SAAS,CAG/B"}

package/dist/indexing/job-resume.js

@@ -0,0 +1,14 @@
+// Resume helpers for the indexing orchestrator (Epic #189, Issue #196). The orchestrator
+// writes `indexing_jobs.status = 'running'` at job-started and transitions to a terminal
+// status at finalize. A run that crashes between those two writes leaves an orphaned row
+// in `running` state — `findResumableJob` is how a follow-up process notices that row.
+//
+// The function is read-only on purpose: we do NOT auto-resume here. Callers (UI surfaces,
+// the eventual `keiko index --resume` CLI) decide whether to re-run, abandon, or alert.
+// The returned `IndexingJobRecord` carries `resume_token` (the lexicographically-greatest
+// embedded `chunk_id` from the prior run) so a resumed run can elect to skip past it.
+import { rowToIndexingJobRecord, selectRunningJobByCapsule } from "./job-persist.js";
+export function findResumableJob(store, capsuleId) {
+    const row = selectRunningJobByCapsule(store._internal.db, capsuleId);
+    return row === undefined ? undefined : rowToIndexingJobRecord(row);
+}

package/dist/indexing/orchestrator.d.ts

@@ -0,0 +1,3 @@
+import { type IndexingEvent, type IndexingOptions } from "./types.js";
+export declare function runIndexingJob(options: IndexingOptions): AsyncIterable<IndexingEvent>;
+//# sourceMappingURL=orchestrator.d.ts.map

package/dist/indexing/orchestrator.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"orchestrator.d.ts","sourceRoot":"","sources":["../../src/indexing/orchestrator.ts"],"names":[],"mappings":"AA0FA,OAAO,EAKL,KAAK,aAAa,EAClB,KAAK,eAAe,EAErB,MAAM,YAAY,CAAC;AAm/CpB,wBAAuB,cAAc,CAAC,OAAO,EAAE,eAAe,GAAG,aAAa,CAAC,aAAa,CAAC,CA2B5F"}