@happyvertical/smrt-ingestion 0.55.5 → 0.55.7

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (85) hide show
  1. package/AGENTS.md +13 -3
  2. package/FEEDBACK.md +117 -0
  3. package/PROPOSALS.md +173 -0
  4. package/README.md +14 -0
  5. package/REVIEW.md +136 -0
  6. package/agents/execution.md +132 -0
  7. package/dist/chunks/execution-CYjtj9_l.js +8031 -0
  8. package/dist/chunks/execution-CYjtj9_l.js.map +1 -0
  9. package/dist/chunks/execution-internal-DWteDHM2.js +7 -0
  10. package/dist/chunks/execution-internal-DWteDHM2.js.map +1 -0
  11. package/dist/chunks/feedback-B_NILilH.js +577 -0
  12. package/dist/chunks/feedback-B_NILilH.js.map +1 -0
  13. package/dist/chunks/proposal-errors-z24VKWnQ.js +301 -0
  14. package/dist/chunks/proposal-errors-z24VKWnQ.js.map +1 -0
  15. package/dist/dto.d.ts +3 -0
  16. package/dist/dto.d.ts.map +1 -1
  17. package/dist/execution-contracts.d.ts +3 -0
  18. package/dist/execution-contracts.d.ts.map +1 -1
  19. package/dist/execution-internal.d.ts +9 -0
  20. package/dist/execution-internal.d.ts.map +1 -0
  21. package/dist/execution.d.ts +23 -1
  22. package/dist/execution.d.ts.map +1 -1
  23. package/dist/execution.js +1 -7614
  24. package/dist/extraction.d.ts.map +1 -1
  25. package/dist/feedback-contracts.d.ts +40 -0
  26. package/dist/feedback-contracts.d.ts.map +1 -0
  27. package/dist/feedback-dto.d.ts +100 -0
  28. package/dist/feedback-dto.d.ts.map +1 -0
  29. package/dist/feedback-internal.d.ts +19 -0
  30. package/dist/feedback-internal.d.ts.map +1 -0
  31. package/dist/feedback.d.ts +53 -0
  32. package/dist/feedback.d.ts.map +1 -0
  33. package/dist/manifest.json +13 -1
  34. package/dist/models.js +1 -1
  35. package/dist/models.js.map +1 -1
  36. package/dist/proposal-contracts.d.ts +89 -0
  37. package/dist/proposal-contracts.d.ts.map +1 -0
  38. package/dist/proposal-dto.d.ts +124 -0
  39. package/dist/proposal-dto.d.ts.map +1 -0
  40. package/dist/proposal-errors.d.ts +5 -0
  41. package/dist/proposal-errors.d.ts.map +1 -0
  42. package/dist/proposal-sdk.d.ts +12 -0
  43. package/dist/proposal-sdk.d.ts.map +1 -0
  44. package/dist/proposal-validation.d.ts +188 -0
  45. package/dist/proposal-validation.d.ts.map +1 -0
  46. package/dist/proposals.d.ts +44 -0
  47. package/dist/proposals.d.ts.map +1 -0
  48. package/dist/proposals.js +964 -0
  49. package/dist/proposals.js.map +1 -0
  50. package/dist/review-dto.d.ts +115 -0
  51. package/dist/review-dto.d.ts.map +1 -0
  52. package/dist/server.d.ts +56 -0
  53. package/dist/server.d.ts.map +1 -1
  54. package/dist/server.js +263 -341
  55. package/dist/server.js.map +1 -1
  56. package/dist/smrt-knowledge.json +87 -12
  57. package/dist/svelte/components/ActionReview.svelte +232 -0
  58. package/dist/svelte/components/ActionReview.svelte.d.ts +15 -0
  59. package/dist/svelte/components/ActionReview.svelte.d.ts.map +1 -0
  60. package/dist/svelte/components/EvidenceViewer.svelte +31 -0
  61. package/dist/svelte/components/EvidenceViewer.svelte.d.ts +8 -0
  62. package/dist/svelte/components/EvidenceViewer.svelte.d.ts.map +1 -0
  63. package/dist/svelte/components/InboxView.svelte +119 -0
  64. package/dist/svelte/components/InboxView.svelte.d.ts +9 -0
  65. package/dist/svelte/components/InboxView.svelte.d.ts.map +1 -0
  66. package/dist/svelte/components/IntakeInbox.svelte +11 -0
  67. package/dist/svelte/components/IntakeInbox.svelte.d.ts +10 -0
  68. package/dist/svelte/components/IntakeInbox.svelte.d.ts.map +1 -0
  69. package/dist/svelte/components/IntakeReview.svelte +11 -0
  70. package/dist/svelte/components/IntakeReview.svelte.d.ts +10 -0
  71. package/dist/svelte/components/IntakeReview.svelte.d.ts.map +1 -0
  72. package/dist/svelte/components/ReviewView.svelte +243 -0
  73. package/dist/svelte/components/ReviewView.svelte.d.ts +9 -0
  74. package/dist/svelte/components/ReviewView.svelte.d.ts.map +1 -0
  75. package/dist/svelte/i18n.d.ts +72 -0
  76. package/dist/svelte/i18n.d.ts.map +1 -0
  77. package/dist/svelte/i18n.js +72 -0
  78. package/dist/svelte/index.d.ts +5 -0
  79. package/dist/svelte/index.d.ts.map +1 -0
  80. package/dist/svelte/index.js +3 -0
  81. package/dist/svelte/view-utils.d.ts +13 -0
  82. package/dist/svelte/view-utils.d.ts.map +1 -0
  83. package/dist/svelte/view-utils.js +96 -0
  84. package/package.json +53 -22
  85. package/dist/execution.js.map +0 -1
package/dist/server.js CHANGED
@@ -1,300 +1,14 @@
1
1
  import "./models.js";
2
- import { a as providerIdentity, i as validateExtractionRequest, n as emptyExtraction, t as ExtractionFailure } from "./chunks/extraction-providers-C66ADDVe.js";
2
+ import { n as generationSnapshotRead } from "./chunks/execution-internal-DWteDHM2.js";
3
+ import { i as proposeDocumentSplits, n as createSDKExtractionAdapter, r as extractAnalysis, t as GenerationSnapshotStaleError } from "./chunks/proposal-errors-z24VKWnQ.js";
3
4
  import { detectEngine, isEmbeddedDatabase, isTransientDatabaseError, withEmbeddedWriteTransaction } from "@happyvertical/smrt-core";
4
5
  import { withTenant } from "@happyvertical/smrt-tenancy";
5
6
  import { createHash, randomUUID } from "node:crypto";
6
7
  import { Asset } from "@happyvertical/smrt-assets";
7
8
  import { SmrtJobCollection } from "@happyvertical/smrt-jobs";
8
- import { fork } from "node:child_process";
9
- import { dirname, join } from "node:path";
10
- import { fileURLToPath } from "node:url";
9
+ import { join } from "node:path";
11
10
  import { constants } from "node:fs";
12
11
  import { lstat, mkdir, open, readFile, readdir, rename, writeFile } from "node:fs/promises";
13
- //#region src/extraction.ts
14
- function createSDKExtractionAdapter(configuration) {
15
- try {
16
- configuration = structuredClone(configuration);
17
- } catch {
18
- throw new Error("Serializable SDK configuration required");
19
- }
20
- const maximum = configuration.maxConcurrentProcesses ?? 1;
21
- if (!Number.isSafeInteger(maximum) || maximum < 1 || maximum > 32) throw new Error("Invalid extraction concurrency");
22
- let running = 0;
23
- if (configuration.nativeMemoryIsolation !== "host-enforced") throw new Error("Native memory isolation is required");
24
- for (const provider of [
25
- configuration.pdf,
26
- configuration.ocr,
27
- configuration.speech,
28
- configuration.vision
29
- ]) if (provider) provider.identity = providerIdentity(provider.identity);
30
- if (configuration.pdf?.provider === "auto" || configuration.ocr?.identity.provider === "auto") throw new Error("Pin extraction providers");
31
- return { async extract(request) {
32
- let latest = emptyExtraction(request);
33
- try {
34
- validateExtractionRequest(request);
35
- } catch (caught) {
36
- if (!(caught instanceof ExtractionFailure)) throw caught;
37
- latest.errors.push({
38
- category: caught.category,
39
- location: { kind: "source" }
40
- });
41
- return latest;
42
- }
43
- if (request.evidence.mediaType.toLowerCase().startsWith("video/")) {
44
- latest.status = "unsupported";
45
- latest.errors.push({
46
- category: "unsupported_type",
47
- location: { kind: "source" }
48
- });
49
- return latest;
50
- }
51
- if (request.signal?.aborted) {
52
- latest.errors.push({
53
- category: "cancelled",
54
- location: { kind: "source" }
55
- });
56
- return latest;
57
- }
58
- if (running >= maximum) {
59
- latest.errors.push({
60
- category: "limit",
61
- location: { kind: "source" }
62
- });
63
- return latest;
64
- }
65
- running++;
66
- return new Promise((resolve) => {
67
- const child = fork(join(dirname(fileURLToPath(import.meta.url)), "extraction-worker.js"), [], {
68
- serialization: "advanced",
69
- stdio: [
70
- "ignore",
71
- "ignore",
72
- "ignore",
73
- "ipc"
74
- ],
75
- execArgv: [`--max-old-space-size=${request.limits.workerHeapMb}`],
76
- detached: process.platform !== "win32"
77
- });
78
- let done = false;
79
- const stop = () => {
80
- try {
81
- if (process.platform !== "win32" && child.pid) process.kill(-child.pid, "SIGKILL");
82
- else child.kill("SIGKILL");
83
- } catch {
84
- child.kill("SIGKILL");
85
- }
86
- };
87
- const finish = (category) => {
88
- if (done) return;
89
- done = true;
90
- running--;
91
- clearTimeout(timer);
92
- request.signal?.removeEventListener("abort", abort);
93
- stop();
94
- if (category) {
95
- latest.errors.push({
96
- category,
97
- location: { kind: "source" }
98
- });
99
- latest.status = latest.segments.length ? "partial" : "failed";
100
- latest.truncated = true;
101
- latest.omitted.push({ kind: "source" });
102
- }
103
- resolve(latest);
104
- };
105
- const abort = () => finish("cancelled");
106
- const timer = setTimeout(() => finish("timeout"), request.limits.timeoutMs);
107
- request.signal?.addEventListener("abort", abort, { once: true });
108
- child.on("error", () => finish("unavailable"));
109
- child.on("exit", () => {
110
- if (!done) finish("unavailable");
111
- });
112
- child.on("message", async (message) => {
113
- if (done) return;
114
- if (message.kind === "authorize" && Number.isSafeInteger(message.id)) {
115
- try {
116
- await request.beforeProviderCall?.();
117
- if (!done) child.send({
118
- kind: "authorized",
119
- id: message.id
120
- });
121
- } catch {
122
- finish("cancelled");
123
- }
124
- return;
125
- }
126
- if ((message.kind === "progress" || message.kind === "result") && message.result) {
127
- if (Buffer.byteLength(JSON.stringify(message.result)) > request.limits.maxOutputBytes + 65536) return finish("limit");
128
- latest = message.result;
129
- if (message.kind === "result") finish();
130
- } else finish("unavailable");
131
- });
132
- const { signal: _signal, beforeProviderCall: _authorize, ...wireRequest } = request;
133
- child.send({
134
- request: wireRequest,
135
- configuration
136
- }, (error) => {
137
- if (error) finish("unavailable");
138
- });
139
- if (request.signal?.aborted) abort();
140
- });
141
- } };
142
- }
143
- function validateAndNormalizeExtractionOutput(result, request) {
144
- const numeric = (value) => Number.isFinite(value) && value >= 0;
145
- const location = (value) => {
146
- if (!value || typeof value !== "object") return false;
147
- if (value.kind === "source") return true;
148
- if (value.kind === "page") return Number.isSafeInteger(value.page) && value.page > 0;
149
- if (value.kind === "pages") return Number.isSafeInteger(value.startPage) && value.startPage > 0 && Number.isSafeInteger(value.endPage) && value.endPage >= value.startPage;
150
- return value.kind === "time" && numeric(value.startMs) && numeric(value.endMs) && value.endMs >= value.startMs;
151
- };
152
- if (!result || ![
153
- "complete",
154
- "partial",
155
- "failed",
156
- "unsupported"
157
- ].includes(result.status) || result.automaticActionEligible !== false || result.configurationRevision !== request.configurationRevision || !result.evidence || Object.entries(request.evidence).some(([key, value]) => result.evidence[key] !== value) || typeof result.truncated !== "boolean" || !Array.isArray(result.capabilities) || result.capabilities.some((capability) => !capability || typeof capability.confidence !== "boolean" || typeof capability.boxes !== "boolean" || ![
158
- "source",
159
- "page",
160
- "time"
161
- ].includes(capability.location) || !["unknown", "reported"].includes(capability.truncation) || !["unknown", "reported"].includes(capability.usage)) || !result.limits || Object.entries(request.limits).some(([key, value]) => result.limits[key] !== value) || !Array.isArray(result.segments) || !Array.isArray(result.errors) || !Array.isArray(result.omitted) || result.status === "complete" && (result.truncated || result.errors.length > 0 || result.omitted.length > 0) || result.omitted.some((value) => !location(value)) || result.segments.some((segment) => typeof segment.text !== "string" || !location(segment.location) || segment.location.kind === "pages" || segment.provenance?.provider === "unlimited-ocr" && (segment.confidence !== null || segment.boxes !== null) || !segment.provenance || [
162
- segment.provenance.provider,
163
- segment.provenance.model,
164
- segment.provenance.version
165
- ].some((value) => typeof value !== "string" || !value) || segment.confidence !== null && (!segment.confidence || !numeric(segment.confidence.value) || segment.confidence.value > 1 || segment.confidence.scale !== "0-1" || segment.confidence.calibration !== "unknown") || segment.boxes !== null && (!Array.isArray(segment.boxes) || segment.boxes.some((box) => typeof box.text !== "string" || ![
166
- box.x,
167
- box.y,
168
- box.width,
169
- box.height
170
- ].every(numeric)))) || result.errors.some((error) => ![
171
- "unsupported_type",
172
- "malformed_output",
173
- "limit",
174
- "timeout",
175
- "unavailable",
176
- "authentication",
177
- "cancelled",
178
- "integrity"
179
- ].includes(error.category) || !location(error.location)) || !result.usage || Object.values(result.usage).some((value) => !numeric(value))) throw new Error("Invalid extraction output");
180
- for (const segment of result.segments) segment.provenance = providerIdentity(segment.provenance);
181
- result.capabilities = result.capabilities.map((capability) => ({
182
- ...providerIdentity(capability),
183
- confidence: capability.confidence,
184
- boxes: capability.boxes,
185
- location: capability.location,
186
- truncation: capability.truncation,
187
- usage: capability.usage
188
- }));
189
- }
190
- async function extractAnalysis(service, lease, adapter, options) {
191
- const snapshot = await service.getAnalysisInput(lease);
192
- const persisted = snapshot.configuration.extraction;
193
- if (!persisted || persisted.configurationRevision !== options.configurationRevision || !persisted.limits || Object.keys(options.limits).some((key) => persisted.limits?.[key] !== options.limits[key]) || Object.keys(persisted.limits).length !== Object.keys(options.limits).length) throw new Error("Extraction configuration mismatch");
194
- const evidence = snapshot.evidence;
195
- const boundOptions = {
196
- ...options,
197
- limits: structuredClone(options.limits)
198
- };
199
- const results = [];
200
- let limited = false;
201
- const publication = (reserve = false) => {
202
- const allComplete = !limited && results.length === evidence.length && results.every((result) => result.status === "complete");
203
- const firstError = limited || reserve ? "limit" : results.flatMap((result) => result.errors)[0]?.category;
204
- return {
205
- status: reserve ? "needs_attention" : allComplete ? "completed" : results.some((result) => result.segments.length) ? "partial" : "needs_attention",
206
- provider: "smrt-ingestion-extraction",
207
- model: "none",
208
- version: "1",
209
- output: {
210
- configurationRevision: options.configurationRevision,
211
- results,
212
- ...limited || reserve ? {
213
- truncated: true,
214
- omittedEvidenceCount: evidence.length - results.length
215
- } : {}
216
- },
217
- usage: {},
218
- ...firstError ? { error: firstError } : {}
219
- };
220
- };
221
- const fits = (output) => Buffer.byteLength(JSON.stringify(output)) <= snapshot.maxOutputBytes;
222
- const minimum = {
223
- status: "needs_attention",
224
- provider: "smrt-ingestion-extraction",
225
- model: "none",
226
- version: "1",
227
- output: {},
228
- usage: {},
229
- error: "limit"
230
- };
231
- if (!fits(minimum)) return service.failAnalysis(lease, "limit");
232
- if (!fits(publication(true))) return service.completeAnalysis(lease, minimum);
233
- for (const part of evidence) {
234
- await service.getAnalysisInput(lease);
235
- const bytes = await service.readEvidence(lease.itemId, part.id);
236
- await service.getAnalysisInput(lease);
237
- const request = {
238
- ...boundOptions,
239
- evidence: part,
240
- bytes
241
- };
242
- const result = await adapter.extract({
243
- ...request,
244
- evidence: { ...part },
245
- limits: { ...request.limits },
246
- bytes: bytes.slice(),
247
- beforeProviderCall: async () => {
248
- await service.getAnalysisInput(lease);
249
- }
250
- });
251
- validateAndNormalizeExtractionOutput(result, request);
252
- results.push(result);
253
- if (!fits(publication(true))) {
254
- limited = true;
255
- result.status = "partial";
256
- result.truncated = true;
257
- result.errors.push({
258
- category: "limit",
259
- location: { kind: "source" }
260
- });
261
- while (result.segments.length && !fits(publication(true))) {
262
- const removed = result.segments.pop();
263
- if (removed) result.omitted.push(removed.location);
264
- }
265
- if (!fits(publication(true))) results.pop();
266
- }
267
- await service.getAnalysisInput(lease);
268
- if (limited || options.signal?.aborted) break;
269
- }
270
- return service.completeAnalysis(lease, publication());
271
- }
272
- function proposeDocumentSplits(result, revision, groups) {
273
- if (!Number.isSafeInteger(revision) || revision < 1 || !groups.length) throw new Error("Invalid split revision");
274
- const available = new Set(result.segments.flatMap((segment) => segment.location.kind === "page" ? [segment.location.page] : []));
275
- const seen = /* @__PURE__ */ new Set();
276
- for (const group of groups) {
277
- if (!group.length) throw new Error("Empty split");
278
- for (const page of group) {
279
- if (!Number.isSafeInteger(page) || !available.has(page) || seen.has(page)) throw new Error("Invalid split page");
280
- seen.add(page);
281
- }
282
- if (group.some((page, index) => index > 0 && page <= group[index - 1])) throw new Error("Unordered split");
283
- }
284
- const proposal = {
285
- revision,
286
- evidenceId: result.evidence.id,
287
- inputHash: result.evidence.contentHash,
288
- configurationRevision: result.configurationRevision,
289
- groups: groups.map((pages) => ({ pages: [...pages] })),
290
- status: "proposed"
291
- };
292
- return {
293
- ...proposal,
294
- digest: createHash("sha256").update(JSON.stringify(proposal)).digest("hex")
295
- };
296
- }
297
- //#endregion
298
12
  //#region src/sources/common.ts
299
13
  var SourceInputError = class extends Error {
300
14
  constructor(category) {
@@ -931,6 +645,9 @@ var IngestionService = class {
931
645
  async reconcileAction(actionId) {
932
646
  return (await this.execution()).reconcileAction(actionId);
933
647
  }
648
+ async listReviews(itemId, input = {}) {
649
+ return (await this.execution()).listReviews(itemId, input);
650
+ }
934
651
  async getAction(actionId) {
935
652
  return (await this.execution()).getAction(actionId);
936
653
  }
@@ -943,6 +660,48 @@ var IngestionService = class {
943
660
  async applyPlan(planId) {
944
661
  return (await this.execution()).applyPlan(planId);
945
662
  }
663
+ async proposals() {
664
+ if (!this.options.proposals) throw new Error("Intake proposals are not configured");
665
+ const { IngestionProposalService } = await import("./proposals.js");
666
+ return new IngestionProposalService(this, this.options, this.options.proposals);
667
+ }
668
+ async listHandlers(itemId) {
669
+ return (await this.proposals()).listHandlers(itemId);
670
+ }
671
+ async findCandidates(input) {
672
+ return (await this.proposals()).findCandidates(input);
673
+ }
674
+ async prepareGeneration(itemId, attemptId) {
675
+ return (await this.proposals()).prepareGeneration(itemId, attemptId);
676
+ }
677
+ async generateProposals(lease) {
678
+ return (await this.proposals()).generate(lease);
679
+ }
680
+ async previewGeneratedProposals(input) {
681
+ return (await this.proposals()).preview(input);
682
+ }
683
+ async feedback() {
684
+ const { IntakeFeedbackService } = await import("./chunks/feedback-B_NILilH.js").then((n) => n.n);
685
+ return new IntakeFeedbackService(this.options);
686
+ }
687
+ async recordFeedback(input) {
688
+ return (await this.feedback()).record(input);
689
+ }
690
+ async observeFeedback(input) {
691
+ return (await this.feedback()).observe(input);
692
+ }
693
+ async retrieveFeedback(input) {
694
+ return (await this.proposals()).retrieveFeedback(input);
695
+ }
696
+ async suggestRoutingRule(input) {
697
+ return (await this.feedback()).suggest(input);
698
+ }
699
+ async getRoutingRule(input) {
700
+ return (await this.feedback()).readRule(input);
701
+ }
702
+ async adoptRoutingRule(input) {
703
+ return (await this.feedback()).adopt(input);
704
+ }
946
705
  now() {
947
706
  return this.options.now?.() ?? /* @__PURE__ */ new Date();
948
707
  }
@@ -1329,62 +1088,129 @@ var IngestionService = class {
1329
1088
  }
1330
1089
  /** Append an immutable input revision. Reprocessing never creates actions. */
1331
1090
  async analyze(itemId, configuration, requestKey) {
1091
+ const frozen = structuredClone(configuration);
1092
+ if (Object.hasOwn(frozen, "humanCorrection") && frozen.stage !== "interpret") throw new Error("Human corrections require authenticated split revision");
1093
+ return this.tx((db) => this.appendAnalysis(db, itemId, frozen, requestKey));
1094
+ }
1095
+ async appendAnalysis(db, itemId, configuration, requestKey) {
1332
1096
  nonempty(requestKey);
1333
1097
  const configDigest = digest(configuration);
1334
- return this.tx(async (db) => {
1335
- await this.lock(db, itemId);
1336
- const item = await this.item(itemId, "process", db);
1337
- if (item.receipt_state !== "ready" || item.cancelled) throw new Error("Intake not processable");
1338
- const replay = (await this.rows(db, "intake_analyses", "item_id=?", [itemId])).find((a) => object(a.data).requestKey === requestKey);
1339
- if (replay) {
1340
- if (object(replay.data).configDigest !== configDigest) throw new Error("Analysis request conflict");
1341
- return {
1342
- id: String(replay.id),
1343
- revision: Number(replay.revision)
1344
- };
1345
- }
1346
- const evidence = await this.rows(db, "intake_evidence", "item_id=? AND state='durable'", [itemId]);
1347
- await db.query("UPDATE intake_analysis_attempts SET state='superseded' WHERE tenant_id=? AND confidential_scope_id=? AND item_id=? AND state IN ('queued','running')", this.scope.tenantId, this.scope.confidentialScopeId, itemId);
1348
- await db.query("UPDATE intake_analyses SET state='superseded' WHERE tenant_id=? AND confidential_scope_id=? AND item_id=? AND state IN ('queued','running')", this.scope.tenantId, this.scope.confidentialScopeId, itemId);
1349
- const revision = Number(item.analysis_revision) + 1;
1350
- await db.query("UPDATE intake_dispatches SET state='completed' WHERE tenant_id=? AND confidential_scope_id=? AND item_id=? AND revision<? AND state<>'completed'", this.scope.tenantId, this.scope.confidentialScopeId, itemId, revision);
1351
- const inputs = evidence.map((e) => ({
1352
- id: e.id,
1353
- hash: e.content_hash
1354
- })).sort((a, b) => String(a.id).localeCompare(String(b.id)));
1355
- const [analysis] = await this.insert(db, "intake_analyses", {
1356
- item_id: itemId,
1357
- revision,
1358
- fence: 0,
1359
- attempt_number: 0,
1360
- current_attempt_id: null,
1361
- state: "queued",
1362
- input_digest: digest({
1363
- inputs,
1364
- configuration
1365
- }),
1366
- data: canonical({
1367
- inputs,
1368
- configuration,
1369
- requestKey,
1370
- configDigest
1371
- })
1372
- });
1373
- await db.query("UPDATE intake_items SET analysis_revision=?,processing_state='queued' WHERE tenant_id=? AND confidential_scope_id=? AND id=?", revision, this.scope.tenantId, this.scope.confidentialScopeId, itemId);
1374
- await this.insert(db, "intake_dispatches", {
1375
- item_id: itemId,
1376
- stage: "analyze",
1377
- revision,
1378
- state: "pending",
1379
- job_id: "",
1380
- deliveries: 0,
1381
- data: "{}"
1382
- }, "tenant_id,item_id,stage,revision");
1098
+ await this.lock(db, itemId);
1099
+ const item = await this.item(itemId, "process", db);
1100
+ if (item.receipt_state !== "ready" || item.cancelled) throw new Error("Intake not processable");
1101
+ const replay = (await this.rows(db, "intake_analyses", "item_id=?", [itemId])).find((a) => object(a.data).requestKey === requestKey);
1102
+ if (replay) {
1103
+ if (object(replay.data).configDigest !== configDigest) throw new Error("Analysis request conflict");
1383
1104
  return {
1384
- id: String(analysis.id),
1385
- revision
1105
+ id: String(replay.id),
1106
+ revision: Number(replay.revision)
1386
1107
  };
1108
+ }
1109
+ const evidence = await this.rows(db, "intake_evidence", "item_id=? AND state='durable'", [itemId]);
1110
+ await db.query("UPDATE intake_analysis_attempts SET state='superseded' WHERE tenant_id=? AND confidential_scope_id=? AND item_id=? AND state IN ('queued','running')", this.scope.tenantId, this.scope.confidentialScopeId, itemId);
1111
+ await db.query("UPDATE intake_analyses SET state='superseded' WHERE tenant_id=? AND confidential_scope_id=? AND item_id=? AND state IN ('queued','running')", this.scope.tenantId, this.scope.confidentialScopeId, itemId);
1112
+ const revision = Number(item.analysis_revision) + 1;
1113
+ await db.query("UPDATE intake_dispatches SET state='completed' WHERE tenant_id=? AND confidential_scope_id=? AND item_id=? AND revision<? AND state<>'completed'", this.scope.tenantId, this.scope.confidentialScopeId, itemId, revision);
1114
+ const inputs = evidence.map((e) => ({
1115
+ id: e.id,
1116
+ hash: e.content_hash
1117
+ })).sort((a, b) => String(a.id).localeCompare(String(b.id)));
1118
+ const [analysis] = await this.insert(db, "intake_analyses", {
1119
+ item_id: itemId,
1120
+ revision,
1121
+ fence: 0,
1122
+ attempt_number: 0,
1123
+ current_attempt_id: null,
1124
+ state: "queued",
1125
+ input_digest: digest({
1126
+ inputs,
1127
+ configuration
1128
+ }),
1129
+ data: canonical({
1130
+ inputs,
1131
+ configuration,
1132
+ requestKey,
1133
+ configDigest
1134
+ })
1387
1135
  });
1136
+ await db.query("UPDATE intake_items SET analysis_revision=?,processing_state='queued' WHERE tenant_id=? AND confidential_scope_id=? AND id=?", revision, this.scope.tenantId, this.scope.confidentialScopeId, itemId);
1137
+ await this.insert(db, "intake_dispatches", {
1138
+ item_id: itemId,
1139
+ stage: "analyze",
1140
+ revision,
1141
+ state: "pending",
1142
+ job_id: "",
1143
+ deliveries: 0,
1144
+ data: "{}"
1145
+ }, "tenant_id,item_id,stage,revision");
1146
+ return {
1147
+ id: String(analysis.id),
1148
+ revision
1149
+ };
1150
+ }
1151
+ /** Record an authenticated split correction and queue fresh extraction, never copy model output. */
1152
+ async reviseLogicalSplit(input) {
1153
+ const frozen = structuredClone(input);
1154
+ nonempty(frozen.requestId);
1155
+ integer(frozen.expectedRevision);
1156
+ if (!Array.isArray(frozen.groups) || !frozen.groups.length || frozen.groups.length > 100 || frozen.groups.some((group) => !Array.isArray(group) || !group.length || group.length > 1e3 || group.some((page) => !Number.isSafeInteger(page) || page < 1))) throw new Error("Invalid logical split");
1157
+ const pages = frozen.groups.flat();
1158
+ if (pages.length > 1e3 || new Set(pages).size !== pages.length) throw new Error("Invalid logical split");
1159
+ const requestKey = `split:${frozen.requestId}`;
1160
+ const correction = {
1161
+ ...frozen,
1162
+ actorId: this.scope.actorId,
1163
+ kind: "logical_split"
1164
+ };
1165
+ const replay = await this.tx(async (db) => {
1166
+ await this.lock(db, frozen.itemId);
1167
+ await this.item(frozen.itemId, "process", db);
1168
+ const found = (await this.rows(db, "intake_analyses", "item_id=?", [frozen.itemId])).find((row) => object(row.data).requestKey === requestKey);
1169
+ if (!found) return null;
1170
+ if (digest(object(object(found.data).configuration).humanCorrection) !== digest(correction)) throw new Error("Split request conflict");
1171
+ return {
1172
+ id: String(found.id),
1173
+ revision: Number(found.revision)
1174
+ };
1175
+ });
1176
+ if (replay) {
1177
+ await (await this.proposals())[generationSnapshotRead](frozen.itemId, (work) => this.tx(work), (db) => this.completedAnalysisSnapshot(db, frozen.itemId, frozen.attemptId, false), void 0, true);
1178
+ return replay;
1179
+ }
1180
+ let result;
1181
+ await (await this.proposals())[generationSnapshotRead](frozen.itemId, (work) => this.tx(work), (db) => this.completedAnalysisSnapshot(db, frozen.itemId, frozen.attemptId, true), async (db, current) => {
1182
+ if (current.revision !== frozen.expectedRevision) throw new Error("Stale logical split");
1183
+ const pin = object(object(current.configuration.proposals).source);
1184
+ const source = await this.completedAnalysisSnapshot(db, frozen.itemId, String(pin.attemptId), false);
1185
+ for (const key of [
1186
+ "attemptId",
1187
+ "revision",
1188
+ "inputDigest",
1189
+ "outputDigest",
1190
+ "evidenceDigest"
1191
+ ]) if (pin[key] !== source[key]) throw new Error("Split source changed");
1192
+ const extraction = source.result.output.results;
1193
+ if (!Array.isArray(extraction)) throw new Error("Split source unavailable");
1194
+ const evidence = extraction.find((value) => object(object(value).evidence).id === frozen.evidenceId);
1195
+ if (!evidence) throw new Error("Split evidence unavailable");
1196
+ const observed = /* @__PURE__ */ new Set();
1197
+ for (const value of object(evidence).segments) {
1198
+ const location = object(object(value).location);
1199
+ if (location.kind === "page") observed.add(Number(location.page));
1200
+ if (location.kind === "pages") {
1201
+ const from = Number(location.startPage), to = Number(location.endPage);
1202
+ if (!Number.isSafeInteger(from) || !Number.isSafeInteger(to) || to - from > 1e3) throw new Error("Invalid source range");
1203
+ for (let page = from; page <= to; page++) observed.add(page);
1204
+ }
1205
+ }
1206
+ if (!observed.size || pages.length !== observed.size || pages.some((page) => !observed.has(page))) throw new Error("Split must cover observed pages exactly");
1207
+ result = await this.appendAnalysis(db, frozen.itemId, {
1208
+ ...source.configuration,
1209
+ humanCorrection: correction
1210
+ }, requestKey);
1211
+ }, true);
1212
+ if (!result) throw new Error("Split unavailable");
1213
+ return result;
1388
1214
  }
1389
1215
  /** Allocate a fenced attempt; active leases cannot be stolen before expiry. */
1390
1216
  async claimAnalysis(itemId, revision, workerId) {
@@ -1439,6 +1265,102 @@ var IngestionService = class {
1439
1265
  };
1440
1266
  });
1441
1267
  }
1268
+ async completedAnalysisSnapshot(db, itemId, attemptId, current) {
1269
+ await this.lock(db, itemId);
1270
+ const item = await this.item(itemId, "process", db);
1271
+ if (item.cancelled) throw new Error("Analysis unavailable");
1272
+ if (attemptId === void 0) {
1273
+ if (!current) throw new Error("Historical attempt required");
1274
+ const [latest] = await this.rows(db, "intake_analyses", "item_id=? AND revision=?", [itemId, item.analysis_revision]);
1275
+ if (!latest?.current_attempt_id) throw new Error("Analysis unavailable");
1276
+ attemptId = String(latest.current_attempt_id);
1277
+ }
1278
+ const [attempt] = await this.rows(db, "intake_analysis_attempts", "item_id=? AND id=?", [itemId, attemptId]);
1279
+ if (!attempt || !["completed", "partial"].includes(String(attempt.state)) || !attempt.output_digest) throw new Error("Analysis unavailable");
1280
+ const [analysis] = await this.rows(db, "intake_analyses", "item_id=? AND id=?", [itemId, attempt.analysis_id]);
1281
+ if (!analysis || analysis.current_attempt_id !== attemptId || current && Number(item.analysis_revision) !== Number(analysis.revision)) throw new Error("Analysis unavailable");
1282
+ const data = object(analysis.data);
1283
+ const configuration = object(data.configuration);
1284
+ if (!Array.isArray(data.inputs) || !data.inputs.length || digest({
1285
+ inputs: data.inputs,
1286
+ configuration
1287
+ }) !== analysis.input_digest || digest(configuration) !== data.configDigest) throw new Error("Analysis input integrity");
1288
+ const resultData = object(attempt.data);
1289
+ const result = {
1290
+ status: resultData.status,
1291
+ provider: resultData.provider,
1292
+ model: resultData.model,
1293
+ version: resultData.version,
1294
+ output: resultData.output,
1295
+ usage: resultData.usage,
1296
+ ...Object.hasOwn(resultData, "error") ? { error: resultData.error } : {},
1297
+ ...Object.hasOwn(resultData, "confidence") ? { confidence: resultData.confidence } : {}
1298
+ };
1299
+ if (result.status !== attempt.state || digest(result) !== attempt.output_digest) throw new Error("Analysis output integrity");
1300
+ const evidence = [];
1301
+ for (const input of data.inputs) {
1302
+ const frozen = object(input);
1303
+ const [row] = await this.rows(db, "intake_evidence", "item_id=? AND id=? AND state='durable'", [itemId, frozen.id]);
1304
+ if (!row || row.content_hash !== frozen.hash) throw new Error("Analysis input integrity");
1305
+ evidence.push({
1306
+ id: String(row.id),
1307
+ partId: String(row.part_id),
1308
+ parentEvidenceId: row.parent_evidence_id ? String(row.parent_evidence_id) : null,
1309
+ mediaType: String(row.media_type),
1310
+ contentHash: String(row.content_hash),
1311
+ byteLength: Number(row.byte_length)
1312
+ });
1313
+ }
1314
+ if (new Set(evidence.map((entry) => entry.id)).size !== evidence.length) throw new Error("Analysis input integrity");
1315
+ await this.item(itemId, "process", db);
1316
+ return structuredClone({
1317
+ itemId,
1318
+ attemptId,
1319
+ revision: Number(analysis.revision),
1320
+ inputDigest: String(analysis.input_digest),
1321
+ outputDigest: String(attempt.output_digest),
1322
+ evidenceDigest: digest(data.inputs),
1323
+ configuration,
1324
+ evidence,
1325
+ result
1326
+ });
1327
+ }
1328
+ /** Detached successful current revision; always current-scope/process authorized. */
1329
+ async getCompletedAnalysis(itemId, attemptId) {
1330
+ const snapshot = await this.tx((db) => this.completedAnalysisSnapshot(db, itemId, attemptId, true));
1331
+ await this.point("analysis:snapshot-read");
1332
+ const read = async (db) => {
1333
+ const current = await this.completedAnalysisSnapshot(db, itemId, snapshot.attemptId, true);
1334
+ if (current.inputDigest !== snapshot.inputDigest || current.outputDigest !== snapshot.outputDigest) throw new Error("Analysis changed");
1335
+ return current;
1336
+ };
1337
+ if (snapshot.configuration.stage === "interpret") return (await this.proposals())[generationSnapshotRead](itemId, (work) => this.tx(work), read);
1338
+ return this.tx(read);
1339
+ }
1340
+ /** Historical source is readable only through its frozen pin on a live generation lease. */
1341
+ async getGenerationInput(lease) {
1342
+ const generation = await this.getAnalysisInput(lease);
1343
+ const configuration = generation.configuration;
1344
+ const pin = object(object(configuration.proposals).source);
1345
+ if (configuration.stage !== "interpret" || typeof pin.attemptId !== "string") throw new Error("Invalid generation source");
1346
+ const source = await this.tx((db) => this.completedAnalysisSnapshot(db, lease.itemId, String(pin.attemptId), false));
1347
+ for (const key of [
1348
+ "attemptId",
1349
+ "revision",
1350
+ "inputDigest",
1351
+ "outputDigest",
1352
+ "evidenceDigest"
1353
+ ]) if (pin[key] !== source[key]) throw new Error("Generation source changed");
1354
+ if (source.result.provider !== "smrt-ingestion-extraction" || source.configuration.stage === "interpret" || source.revision !== generation.revision - 1 || digest(configuration.humanCorrection ?? null) !== digest(source.configuration.humanCorrection ?? null) || digest(generation.evidence.map((entry) => ({
1355
+ id: entry.id,
1356
+ hash: entry.contentHash
1357
+ }))) !== source.evidenceDigest) throw new Error("Generation evidence changed");
1358
+ await this.getAnalysisInput(lease);
1359
+ return {
1360
+ generation,
1361
+ source
1362
+ };
1363
+ }
1442
1364
  /** Reload immutable inputs only through the current live processing lease. */
1443
1365
  async getAnalysisInput(lease) {
1444
1366
  return this.tx(async (db) => {
@@ -1753,6 +1675,6 @@ var IngestionService = class {
1753
1675
  }
1754
1676
  };
1755
1677
  //#endregion
1756
- export { EmailSourceAdapter, IngestionService, WatchFolderSourceAdapter, createSDKExtractionAdapter, createSourceDeliveryHandler, createVendorWebhookHandler, extractAnalysis, proposeDocumentSplits };
1678
+ export { EmailSourceAdapter, GenerationSnapshotStaleError, IngestionService, WatchFolderSourceAdapter, createSDKExtractionAdapter, createSourceDeliveryHandler, createVendorWebhookHandler, extractAnalysis, proposeDocumentSplits };
1757
1679
 
1758
1680
  //# sourceMappingURL=server.js.map