@evomap/evolver-core 2.0.0-beta.19 → 2.0.0-beta.22

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (160) hide show
  1. package/assets/gep/genes.jsonl +5 -5
  2. package/dist/algo/bans.d.ts +10 -1
  3. package/dist/algo/bans.js +56 -6
  4. package/dist/algo/candidateAssembly.d.ts +25 -2
  5. package/dist/algo/candidateAssembly.js +143 -20
  6. package/dist/algo/capabilityCandidates.js +10 -0
  7. package/dist/algo/cycleEngine.d.ts +40 -5
  8. package/dist/algo/cycleEngine.js +277 -37
  9. package/dist/algo/evolutionEvent.d.ts +26 -0
  10. package/dist/algo/evolutionEvent.js +32 -0
  11. package/dist/algo/exploration.d.ts +7 -0
  12. package/dist/algo/exploration.js +16 -3
  13. package/dist/algo/geneHealth.d.ts +34 -1
  14. package/dist/algo/geneHealth.js +42 -0
  15. package/dist/algo/geneIntake.d.ts +46 -1
  16. package/dist/algo/geneIntake.js +121 -10
  17. package/dist/algo/geneSelection.d.ts +115 -6
  18. package/dist/algo/geneSelection.js +459 -32
  19. package/dist/algo/index.d.ts +4 -1
  20. package/dist/algo/index.js +4 -1
  21. package/dist/algo/kautoProjection.d.ts +41 -0
  22. package/dist/algo/kautoProjection.js +95 -0
  23. package/dist/algo/kautoValidator.d.ts +68 -0
  24. package/dist/algo/kautoValidator.js +256 -0
  25. package/dist/algo/orchestrator.d.ts +14 -1
  26. package/dist/algo/orchestrator.js +9 -2
  27. package/dist/algo/solidify.d.ts +11 -2
  28. package/dist/algo/solidify.js +37 -7
  29. package/dist/algo/ucb1.d.ts +53 -0
  30. package/dist/algo/ucb1.js +156 -0
  31. package/dist/assetrepair/hubRejection.d.ts +12 -0
  32. package/dist/assetrepair/hubRejection.js +109 -0
  33. package/dist/assetrepair/index.d.ts +2 -0
  34. package/dist/assetrepair/index.js +2 -0
  35. package/dist/assetrepair/repair.d.ts +33 -0
  36. package/dist/assetrepair/repair.js +155 -0
  37. package/dist/assetstore/assetSidecarRecords.js +5 -1
  38. package/dist/assetstore/assetStoreHealth.d.ts +2 -3
  39. package/dist/assetstore/assetStoreHealth.js +4 -10
  40. package/dist/assetstore/foreignJsonlSource.d.ts +48 -0
  41. package/dist/assetstore/foreignJsonlSource.js +150 -0
  42. package/dist/assetstore/index.d.ts +3 -0
  43. package/dist/assetstore/index.js +3 -0
  44. package/dist/assetstore/learningHistory.js +3 -3
  45. package/dist/assetstore/localAssetStoreSnapshot.d.ts +51 -0
  46. package/dist/assetstore/localAssetStoreSnapshot.js +329 -0
  47. package/dist/assetstore/localJsonl.d.ts +8 -0
  48. package/dist/assetstore/localJsonl.js +128 -3
  49. package/dist/assetstore/provenance.d.ts +38 -3
  50. package/dist/assetstore/provenance.js +103 -45
  51. package/dist/assetstore/provider.d.ts +4 -0
  52. package/dist/assetstore/reviewFilter.d.ts +19 -1
  53. package/dist/assetstore/reviewFilter.js +36 -0
  54. package/dist/assetstore/unionReadStore.d.ts +25 -0
  55. package/dist/assetstore/unionReadStore.js +119 -0
  56. package/dist/benchmark/antiGeneBenchmark.d.ts +2 -0
  57. package/dist/benchmark/antiGeneBenchmark.js +4 -3
  58. package/dist/benchmark/antiGeneRollout.d.ts +2 -0
  59. package/dist/benchmark/antiGeneRollout.js +4 -3
  60. package/dist/benchmark/index.d.ts +2 -1
  61. package/dist/benchmark/index.js +2 -1
  62. package/dist/benchmark/selectionFlatAbstention.d.ts +152 -0
  63. package/dist/benchmark/selectionFlatAbstention.js +481 -0
  64. package/dist/bootstrap/envFingerprint.d.ts +9 -0
  65. package/dist/bootstrap/envFingerprint.js +5 -0
  66. package/dist/bootstrap/index.d.ts +2 -1
  67. package/dist/bootstrap/index.js +2 -1
  68. package/dist/bootstrap/lifecycleBootstrap.d.ts +111 -0
  69. package/dist/bootstrap/lifecycleBootstrap.js +433 -0
  70. package/dist/bootstrap/v1EnvCompat.d.ts +3 -0
  71. package/dist/bootstrap/v1EnvCompat.js +48 -1
  72. package/dist/events/eventArchive.d.ts +2 -0
  73. package/dist/events/eventArchive.js +13 -3
  74. package/dist/events/eventSchema.d.ts +7 -7
  75. package/dist/events/eventStore.d.ts +2 -0
  76. package/dist/events/eventStore.js +5 -1
  77. package/dist/events/ingest.d.ts +2 -1
  78. package/dist/events/ingest.js +6 -0
  79. package/dist/exec/autoExec.d.ts +30 -3
  80. package/dist/exec/autoExec.js +294 -29
  81. package/dist/exec/autonomousCycle.d.ts +7 -0
  82. package/dist/exec/autonomousCycle.js +2 -0
  83. package/dist/exec/claudeBridge.d.ts +28 -6
  84. package/dist/exec/claudeBridge.js +411 -258
  85. package/dist/exec/executionBinding.d.ts +414 -0
  86. package/dist/exec/executionBinding.js +588 -0
  87. package/dist/exec/index.d.ts +1 -0
  88. package/dist/exec/index.js +1 -0
  89. package/dist/exec/proofOfWork.d.ts +1 -1
  90. package/dist/exec/proofOfWork.js +2 -2
  91. package/dist/exec/runnerRegistry.d.ts +5 -0
  92. package/dist/exec/runnerRegistry.js +54 -14
  93. package/dist/hub/capability.d.ts +40 -1
  94. package/dist/hub/conversationDistiller.d.ts +19 -0
  95. package/dist/hub/conversationDistiller.js +115 -37
  96. package/dist/hub/fake.d.ts +1 -0
  97. package/dist/hub/fake.js +1 -0
  98. package/dist/hub/index.d.ts +1 -0
  99. package/dist/hub/index.js +1 -0
  100. package/dist/hub/recipeCompose.d.ts +27 -0
  101. package/dist/hub/recipeCompose.js +90 -0
  102. package/dist/hub/sanitize.js +119 -5
  103. package/dist/index.d.ts +2 -0
  104. package/dist/index.js +2 -0
  105. package/dist/mailbox/ipcServer.d.ts +2 -0
  106. package/dist/mailbox/ipcServer.js +20 -0
  107. package/dist/mailbox/store.d.ts +52 -2
  108. package/dist/mailbox/store.js +546 -81
  109. package/dist/modelCompatibility.d.ts +164 -0
  110. package/dist/modelCompatibility.js +309 -0
  111. package/dist/observers/valueDigestObserver.d.ts +9 -0
  112. package/dist/observers/valueDigestObserver.js +35 -2
  113. package/dist/ops/cleanup.js +1 -1
  114. package/dist/ops/evolutionGraphProjection.d.ts +20 -0
  115. package/dist/ops/evolutionGraphProjection.js +315 -0
  116. package/dist/ops/index.d.ts +2 -1
  117. package/dist/ops/index.js +2 -1
  118. package/dist/ops/selfUpdate.d.ts +8 -0
  119. package/dist/ops/selfUpdate.js +24 -8
  120. package/dist/ops/valueOutreach.d.ts +3 -1
  121. package/dist/ops/valueOutreach.js +5 -1
  122. package/dist/personality/schema.d.ts +18 -18
  123. package/dist/schema/evolutionGraph.d.ts +784 -0
  124. package/dist/schema/evolutionGraph.js +187 -0
  125. package/dist/schema/index.d.ts +1 -0
  126. package/dist/schema/index.js +1 -0
  127. package/dist/schema/proofOfWork.d.ts +125 -6
  128. package/dist/schema/proofOfWork.js +102 -4
  129. package/dist/shadow/shadowHub.js +1 -0
  130. package/dist/signals/expand.d.ts +15 -1
  131. package/dist/signals/expand.js +148 -0
  132. package/dist/signals/index.d.ts +3 -1
  133. package/dist/signals/index.js +3 -1
  134. package/dist/signals/scopeVocabulary.d.ts +75 -0
  135. package/dist/signals/scopeVocabulary.js +91 -0
  136. package/dist/signals/taskDomain.d.ts +22 -0
  137. package/dist/signals/taskDomain.js +43 -0
  138. package/dist/trace/learningTrace.d.ts +7 -0
  139. package/dist/trace/learningTrace.js +22 -0
  140. package/dist/trace/trajectory.d.ts +8 -0
  141. package/dist/trace/trajectory.js +14 -2
  142. package/dist/util/fileLock.d.ts +92 -5
  143. package/dist/util/fileLock.js +809 -70
  144. package/dist/util/index.d.ts +2 -1
  145. package/dist/util/index.js +1 -1
  146. package/dist/verify/index.d.ts +2 -1
  147. package/dist/verify/index.js +1 -1
  148. package/dist/verify/sandboxRunner.d.ts +2 -0
  149. package/dist/verify/sandboxRunner.js +154 -10
  150. package/dist/verify/sandboxedValidation.d.ts +7 -2
  151. package/dist/verify/sandboxedValidation.js +102 -24
  152. package/dist/verify/validation.d.ts +24 -3
  153. package/dist/verify/validation.js +141 -24
  154. package/dist/wire/geneHints.d.ts +88 -16
  155. package/dist/wire/geneHints.js +124 -15
  156. package/dist/wire/index.d.ts +8 -3
  157. package/dist/wire/index.js +2 -2
  158. package/dist/wire/schemaGate.d.ts +21 -0
  159. package/dist/wire/schemaGate.js +119 -10
  160. package/package.json +5 -4
@@ -1,5 +1,7 @@
1
- import { mkdirSync, existsSync, openSync, readSync, closeSync } from 'node:fs';
2
- import { dirname } from 'node:path';
1
+ import { createHash } from 'node:crypto';
2
+ import { chmodSync, closeSync, constants, existsSync, fstatSync, fsyncSync, lstatSync, mkdirSync, mkdtempSync, openSync, readSync, rmSync, writeSync, } from 'node:fs';
3
+ import { tmpdir } from 'node:os';
4
+ import { basename, dirname, join } from 'node:path';
3
5
  import { createRequire } from 'node:module';
4
6
  import { ulid as makeUlid } from 'ulid';
5
7
  import { createEnvelope, ENVELOPE_SCHEMA_VERSION, normalizePriority, sanitizePayload, specForType, } from './envelope.js';
@@ -17,10 +19,231 @@ function openSqliteDatabase(path, options = {}) {
17
19
  const { DatabaseSync } = nodeRequire('node:sqlite');
18
20
  return options.readOnly ? new DatabaseSync(path, { readOnly: true }) : new DatabaseSync(path);
19
21
  }
22
+ const MAILBOX_PREVIEW_MAX_FILE_BYTES = 256 * 1024 * 1024;
23
+ const MAILBOX_PREVIEW_MAX_TOTAL_BYTES = 512 * 1024 * 1024;
24
+ const MAILBOX_PREVIEW_COPY_BUFFER_BYTES = 64 * 1024;
25
+ function mailboxPreviewError(code) {
26
+ return new Error(`mailbox_preview_${code}`);
27
+ }
28
+ function mailboxPreviewNoFollowFlag() {
29
+ return constants['O_NOFOLLOW'] ?? 0;
30
+ }
31
+ function mailboxPreviewStat(path) {
32
+ let stat;
33
+ try {
34
+ stat = lstatSync(path, { bigint: true });
35
+ }
36
+ catch (error) {
37
+ if (error.code === 'ENOENT')
38
+ return null;
39
+ throw mailboxPreviewError('target_unsafe');
40
+ }
41
+ if (!stat.isFile() || stat.isSymbolicLink() || stat.nlink > 1n) {
42
+ throw mailboxPreviewError('target_unsafe');
43
+ }
44
+ return stat;
45
+ }
46
+ function sameMailboxPreviewSnapshot(left, right) {
47
+ const hasStableFileId = left.dev !== 0n || left.ino !== 0n || right.dev !== 0n || right.ino !== 0n;
48
+ const sameIdentity = hasStableFileId
49
+ ? left.dev === right.dev && left.ino === right.ino
50
+ : left.birthtimeNs === right.birthtimeNs && left.mode === right.mode;
51
+ return sameIdentity
52
+ && left.mode === right.mode
53
+ && left.size === right.size
54
+ && left.mtimeNs === right.mtimeNs
55
+ && left.ctimeNs === right.ctimeNs;
56
+ }
57
+ function assertMailboxPreviewSize(stat) {
58
+ if (stat.size > BigInt(MAILBOX_PREVIEW_MAX_FILE_BYTES)
59
+ || stat.size > BigInt(Number.MAX_SAFE_INTEGER)) {
60
+ throw mailboxPreviewError('target_too_large');
61
+ }
62
+ return Number(stat.size);
63
+ }
64
+ function writeMailboxPreviewChunk(fd, chunk, position) {
65
+ let offset = 0;
66
+ while (offset < chunk.length) {
67
+ const written = writeSync(fd, chunk, offset, chunk.length - offset, position + offset);
68
+ if (written <= 0)
69
+ throw mailboxPreviewError('snapshot_write_failed');
70
+ offset += written;
71
+ }
72
+ }
73
+ function copyStableMailboxPreviewFile(sourcePath, destinationPath, expected) {
74
+ const expectedBytes = assertMailboxPreviewSize(expected);
75
+ let sourceFd;
76
+ let destinationFd;
77
+ try {
78
+ sourceFd = openSync(sourcePath, constants.O_RDONLY | mailboxPreviewNoFollowFlag());
79
+ const opened = fstatSync(sourceFd, { bigint: true });
80
+ if (!opened.isFile() || !sameMailboxPreviewSnapshot(expected, opened)) {
81
+ throw mailboxPreviewError('target_changed');
82
+ }
83
+ destinationFd = openSync(destinationPath, constants.O_WRONLY | constants.O_CREAT | constants.O_EXCL | mailboxPreviewNoFollowFlag(), 0o600);
84
+ const digest = createHash('sha256');
85
+ const buffer = Buffer.allocUnsafe(MAILBOX_PREVIEW_COPY_BUFFER_BYTES);
86
+ let total = 0;
87
+ for (;;) {
88
+ const bytesRead = readSync(sourceFd, buffer, 0, buffer.length, null);
89
+ if (bytesRead === 0)
90
+ break;
91
+ if (total > MAILBOX_PREVIEW_MAX_FILE_BYTES - bytesRead) {
92
+ throw mailboxPreviewError('target_too_large');
93
+ }
94
+ const chunk = buffer.subarray(0, bytesRead);
95
+ digest.update(chunk);
96
+ writeMailboxPreviewChunk(destinationFd, chunk, total);
97
+ total += bytesRead;
98
+ }
99
+ fsyncSync(destinationFd);
100
+ const afterDescriptor = fstatSync(sourceFd, { bigint: true });
101
+ const afterPath = mailboxPreviewStat(sourcePath);
102
+ if (total !== expectedBytes
103
+ || !sameMailboxPreviewSnapshot(opened, afterDescriptor)
104
+ || !afterPath
105
+ || !sameMailboxPreviewSnapshot(afterDescriptor, afterPath)) {
106
+ throw mailboxPreviewError('target_changed');
107
+ }
108
+ return Object.freeze({ stat: afterDescriptor, digest: digest.digest('hex'), bytes: total });
109
+ }
110
+ catch (error) {
111
+ if (error instanceof Error && error.message.startsWith('mailbox_preview_'))
112
+ throw error;
113
+ throw mailboxPreviewError('snapshot_failed');
114
+ }
115
+ finally {
116
+ if (sourceFd !== undefined)
117
+ closeSync(sourceFd);
118
+ if (destinationFd !== undefined)
119
+ closeSync(destinationFd);
120
+ }
121
+ }
122
+ function hashStableMailboxPreviewFile(path, expected) {
123
+ const before = mailboxPreviewStat(path);
124
+ if (!before || !sameMailboxPreviewSnapshot(expected.stat, before)) {
125
+ throw mailboxPreviewError('target_changed');
126
+ }
127
+ assertMailboxPreviewSize(before);
128
+ let fd;
129
+ try {
130
+ fd = openSync(path, constants.O_RDONLY | mailboxPreviewNoFollowFlag());
131
+ const opened = fstatSync(fd, { bigint: true });
132
+ if (!opened.isFile() || !sameMailboxPreviewSnapshot(before, opened)) {
133
+ throw mailboxPreviewError('target_changed');
134
+ }
135
+ const digest = createHash('sha256');
136
+ const buffer = Buffer.allocUnsafe(MAILBOX_PREVIEW_COPY_BUFFER_BYTES);
137
+ let total = 0;
138
+ for (;;) {
139
+ const bytesRead = readSync(fd, buffer, 0, buffer.length, null);
140
+ if (bytesRead === 0)
141
+ break;
142
+ if (total > MAILBOX_PREVIEW_MAX_FILE_BYTES - bytesRead) {
143
+ throw mailboxPreviewError('target_too_large');
144
+ }
145
+ digest.update(buffer.subarray(0, bytesRead));
146
+ total += bytesRead;
147
+ }
148
+ const afterDescriptor = fstatSync(fd, { bigint: true });
149
+ const afterPath = mailboxPreviewStat(path);
150
+ if (total !== expected.bytes
151
+ || digest.digest('hex') !== expected.digest
152
+ || !sameMailboxPreviewSnapshot(opened, afterDescriptor)
153
+ || !afterPath
154
+ || !sameMailboxPreviewSnapshot(afterDescriptor, afterPath)) {
155
+ throw mailboxPreviewError('target_changed');
156
+ }
157
+ }
158
+ catch (error) {
159
+ if (error instanceof Error && error.message.startsWith('mailbox_preview_'))
160
+ throw error;
161
+ throw mailboxPreviewError('target_changed');
162
+ }
163
+ finally {
164
+ if (fd !== undefined)
165
+ closeSync(fd);
166
+ }
167
+ }
168
+ function assertMailboxRollbackJournalAbsent(mailboxPath) {
169
+ try {
170
+ lstatSync(`${mailboxPath}-journal`);
171
+ }
172
+ catch (error) {
173
+ if (error.code === 'ENOENT')
174
+ return;
175
+ throw mailboxPreviewError('target_unsafe');
176
+ }
177
+ throw mailboxPreviewError('hot_rollback_journal');
178
+ }
179
+ function assertMailboxPreviewTargetAbsent(mailboxPath) {
180
+ for (const suffix of ['-wal', '-shm', '-journal']) {
181
+ try {
182
+ lstatSync(`${mailboxPath}${suffix}`);
183
+ }
184
+ catch (error) {
185
+ if (error.code === 'ENOENT')
186
+ continue;
187
+ throw mailboxPreviewError('target_unsafe');
188
+ }
189
+ throw mailboxPreviewError('target_unsafe');
190
+ }
191
+ if (mailboxPreviewStat(mailboxPath))
192
+ throw mailboxPreviewError('target_changed');
193
+ }
194
+ function stableMailboxPreviewCopy(mailboxPath) {
195
+ const databaseStat = mailboxPreviewStat(mailboxPath);
196
+ if (!databaseStat)
197
+ throw mailboxPreviewError('target_changed');
198
+ assertMailboxRollbackJournalAbsent(mailboxPath);
199
+ const walPath = `${mailboxPath}-wal`;
200
+ const walStat = mailboxPreviewStat(walPath);
201
+ const totalBytes = databaseStat.size + (walStat?.size ?? 0n);
202
+ if (totalBytes > BigInt(MAILBOX_PREVIEW_MAX_TOTAL_BYTES)) {
203
+ throw mailboxPreviewError('target_too_large');
204
+ }
205
+ const snapshotDir = mkdtempSync(join(tmpdir(), 'evolver-mailbox-preview-'));
206
+ const snapshotPath = join(snapshotDir, basename(mailboxPath));
207
+ try {
208
+ chmodSync(snapshotDir, 0o700);
209
+ const databaseSnapshot = copyStableMailboxPreviewFile(mailboxPath, snapshotPath, databaseStat);
210
+ const walSnapshot = walStat
211
+ ? copyStableMailboxPreviewFile(walPath, `${snapshotPath}-wal`, walStat)
212
+ : null;
213
+ assertMailboxRollbackJournalAbsent(mailboxPath);
214
+ const currentWal = mailboxPreviewStat(walPath);
215
+ if ((walSnapshot === null) !== (currentWal === null)) {
216
+ throw mailboxPreviewError('target_changed');
217
+ }
218
+ hashStableMailboxPreviewFile(mailboxPath, databaseSnapshot);
219
+ if (walSnapshot)
220
+ hashStableMailboxPreviewFile(walPath, walSnapshot);
221
+ return { dir: snapshotDir, path: snapshotPath };
222
+ }
223
+ catch (error) {
224
+ try {
225
+ rmSync(snapshotDir, { recursive: true, force: true });
226
+ }
227
+ catch { /* preserve primary error */ }
228
+ throw error;
229
+ }
230
+ }
20
231
  export const MAX_ATTEMPTS = 5;
21
232
  export const DEFAULT_IMPORT_JSONL_MAX_LINE_BYTES = 16 * 1024 * 1024;
233
+ export const DEFAULT_IMPORT_JSONL_MAX_BYTES = 128 * 1024 * 1024;
234
+ export const DEFAULT_IMPORT_JSONL_MAX_RECORDS = 100_000;
22
235
  const JSONL_READ_CHUNK_BYTES = 64 * 1024;
23
236
  export const MAILBOX_CLAIM_OWNER = Symbol('mailboxClaimOwner');
237
+ export class MailboxJsonlImportError extends Error {
238
+ code;
239
+ line;
240
+ constructor(code, line, message) {
241
+ super(`Mailbox JSONL line ${line}: ${message}`);
242
+ this.name = 'MailboxJsonlImportError';
243
+ this.code = code;
244
+ this.line = line;
245
+ }
246
+ }
24
247
  export function mailboxClaimOwner(envelope) {
25
248
  return envelope[MAILBOX_CLAIM_OWNER];
26
249
  }
@@ -97,12 +320,12 @@ function recoverImportedStatus(type, requested) {
97
320
  // with status=in_flight and leasedUntil=NULL, which SQLite will never consider expired or claimable.
98
321
  return requested === 'in_flight' ? 'pending' : requested;
99
322
  }
100
- function normalizeImportedEnvelope(record) {
323
+ function normalizeImportedEnvelope(record, now = Date.now()) {
101
324
  const id = nonEmptyString(record['id']);
102
325
  const type = nonEmptyString(record['type']);
103
326
  if (!id || !type)
104
327
  return undefined;
105
- const createdAt = finiteNumber(record['createdAt']) ?? finiteNumber(record['created_at']) ?? Date.now();
328
+ const createdAt = finiteNumber(record['createdAt']) ?? finiteNumber(record['created_at']) ?? now;
106
329
  const spec = specForType(type);
107
330
  const directionValue = record['direction'];
108
331
  const direction = directionValue === 'outbound' || directionValue === 'inbound' || directionValue === 'local'
@@ -157,49 +380,120 @@ function normalizeImportedEnvelope(record) {
157
380
  lastError: nullableStringField(record, 'lastError', 'last_error', 'error') ?? null,
158
381
  };
159
382
  }
160
- function forEachJsonlRecord(source, onRecord) {
161
- if (typeof source === 'string' && !existsSync(source))
162
- return;
163
- const maxLineBytes = Number.isFinite(Number(process.env['EVOLVER_IMPORT_JSONL_MAX_LINE_BYTES']))
164
- && Number(process.env['EVOLVER_IMPORT_JSONL_MAX_LINE_BYTES']) > 0
165
- ? Math.floor(Number(process.env['EVOLVER_IMPORT_JSONL_MAX_LINE_BYTES']))
166
- : DEFAULT_IMPORT_JSONL_MAX_LINE_BYTES;
383
+ function positiveInteger(value, fallback) {
384
+ const parsed = Number(value);
385
+ return Number.isFinite(parsed) && parsed > 0 ? Math.floor(parsed) : fallback;
386
+ }
387
+ function deepFreeze(value) {
388
+ if (value !== null && typeof value === 'object' && !Object.isFrozen(value)) {
389
+ for (const nested of Object.values(value))
390
+ deepFreeze(nested);
391
+ Object.freeze(value);
392
+ }
393
+ return value;
394
+ }
395
+ const mailboxFatalUtf8Decoder = new TextDecoder('utf-8', { fatal: true });
396
+ function decodeMailboxImportLine(bytes, line) {
397
+ try {
398
+ return mailboxFatalUtf8Decoder.decode(bytes);
399
+ }
400
+ catch {
401
+ throw new MailboxJsonlImportError('invalid_utf8', line, 'invalid UTF-8');
402
+ }
403
+ }
404
+ /**
405
+ * Parse and normalize a V1 mailbox journal without opening or mutating a V2 mailbox.
406
+ * The returned batch is sealed so dry-run and apply can consume the exact same records.
407
+ */
408
+ export function prepareMailboxJsonlImport(source, options = {}) {
409
+ if (typeof source === 'string' && !existsSync(source)) {
410
+ return Object.freeze({
411
+ records: Object.freeze([]),
412
+ sourceRecords: 0,
413
+ messageCandidates: 0,
414
+ uniqueMessageCandidates: 0,
415
+ updateCandidates: 0,
416
+ });
417
+ }
418
+ const maxLineBytes = positiveInteger(options.maxLineBytes ?? process.env['EVOLVER_IMPORT_JSONL_MAX_LINE_BYTES'], DEFAULT_IMPORT_JSONL_MAX_LINE_BYTES);
419
+ const maxRecords = positiveInteger(options.maxRecords ?? process.env['EVOLVER_IMPORT_JSONL_MAX_RECORDS'], DEFAULT_IMPORT_JSONL_MAX_RECORDS);
420
+ const maxBytes = positiveInteger(options.maxBytes ?? process.env['EVOLVER_IMPORT_JSONL_MAX_BYTES'], DEFAULT_IMPORT_JSONL_MAX_BYTES);
421
+ const now = Number.isFinite(options.now) ? Number(options.now) : Date.now();
167
422
  const buf = Buffer.allocUnsafe(JSONL_READ_CHUNK_BYTES);
423
+ const records = [];
424
+ const messageIds = new Set();
425
+ let sourceRecords = 0;
426
+ let messageCandidates = 0;
427
+ let updateCandidates = 0;
168
428
  let fd;
169
429
  let ownsFd = false;
170
430
  let position = 0;
171
431
  let parts = [];
172
432
  let lineBytes = 0;
173
- let dropping = false;
174
- const reset = () => { parts = []; lineBytes = 0; dropping = false; };
433
+ let lineNumber = 1;
434
+ let overlong = false;
435
+ const reset = () => { parts = []; lineBytes = 0; overlong = false; };
175
436
  const append = (segment) => {
176
- if (dropping || segment.length === 0)
437
+ if (overlong || segment.length === 0)
177
438
  return;
178
439
  if (lineBytes + segment.length > maxLineBytes) {
179
440
  reset();
180
- dropping = true;
441
+ overlong = true;
181
442
  return;
182
443
  }
183
444
  parts.push(Buffer.from(segment));
184
445
  lineBytes += segment.length;
185
446
  };
186
447
  const finish = () => {
187
- if (dropping || lineBytes === 0) {
448
+ const currentLine = lineNumber;
449
+ lineNumber += 1;
450
+ if (overlong) {
451
+ reset();
452
+ throw new MailboxJsonlImportError('line_too_large', currentLine, `record exceeds ${maxLineBytes} bytes`);
453
+ }
454
+ if (lineBytes === 0) {
188
455
  reset();
189
456
  return;
190
457
  }
191
- const text = Buffer.concat(parts, lineBytes).toString('utf8').trim();
458
+ const text = decodeMailboxImportLine(Buffer.concat(parts, lineBytes), currentLine).trim();
192
459
  reset();
193
460
  if (!text)
194
461
  return;
462
+ sourceRecords += 1;
463
+ if (sourceRecords > maxRecords) {
464
+ throw new MailboxJsonlImportError('too_many_records', currentLine, `journal exceeds ${maxRecords} records`);
465
+ }
195
466
  let record;
196
467
  try {
197
468
  record = JSON.parse(text);
198
469
  }
199
470
  catch {
200
- return; /* skip corrupt JSON */
471
+ throw new MailboxJsonlImportError('invalid_json', currentLine, 'invalid JSON');
472
+ }
473
+ const row = asRow(record);
474
+ if (!row) {
475
+ throw new MailboxJsonlImportError('invalid_record', currentLine, 'record must be a JSON object');
201
476
  }
202
- onRecord(record);
477
+ if (Object.prototype.hasOwnProperty.call(row, '_op')) {
478
+ if (row['_op'] !== 'update') {
479
+ throw new MailboxJsonlImportError('unsupported_operation', currentLine, 'unsupported _op value');
480
+ }
481
+ const id = nonEmptyString(row['id']);
482
+ const fields = asRow(row['fields']);
483
+ if (!id || !fields) {
484
+ throw new MailboxJsonlImportError('invalid_update', currentLine, 'update requires a non-empty id and object fields');
485
+ }
486
+ records.push(deepFreeze({ kind: 'update', id, fields }));
487
+ updateCandidates += 1;
488
+ return;
489
+ }
490
+ const envelope = normalizeImportedEnvelope(row, now);
491
+ if (!envelope) {
492
+ throw new MailboxJsonlImportError('invalid_message', currentLine, 'message requires a non-empty id, type, and valid direction');
493
+ }
494
+ records.push(deepFreeze({ kind: 'message', envelope }));
495
+ messageCandidates += 1;
496
+ messageIds.add(envelope.id);
203
497
  };
204
498
  try {
205
499
  if (typeof source === 'number') {
@@ -214,6 +508,9 @@ function forEachJsonlRecord(source, onRecord) {
214
508
  if (bytes <= 0)
215
509
  break;
216
510
  position += bytes;
511
+ if (position > maxBytes) {
512
+ throw new MailboxJsonlImportError('journal_too_large', lineNumber, `journal exceeds ${maxBytes} bytes`);
513
+ }
217
514
  let start = 0;
218
515
  for (let i = 0; i < bytes; i += 1) {
219
516
  if (buf[i] !== 0x0a)
@@ -231,6 +528,13 @@ function forEachJsonlRecord(source, onRecord) {
231
528
  if (ownsFd && fd !== undefined)
232
529
  closeSync(fd);
233
530
  }
531
+ return Object.freeze({
532
+ records: Object.freeze(records),
533
+ sourceRecords,
534
+ messageCandidates,
535
+ uniqueMessageCandidates: messageIds.size,
536
+ updateCandidates,
537
+ });
234
538
  }
235
539
  function rowToEnvelope(r) {
236
540
  return {
@@ -249,6 +553,172 @@ function rowToEnvelope(r) {
249
553
  lastError: r['lastError'] ?? null,
250
554
  };
251
555
  }
556
+ function applyImportedV1UpdateToEnvelope(current, fields) {
557
+ const requestedStatus = importedStatus(fields['status']) ?? current.status;
558
+ const status = recoverImportedStatus(current.type, requestedStatus);
559
+ const attempts = Math.max(0, Math.floor(finiteNumber(fields['attempts']) ?? finiteNumber(fields['retry_count']) ?? current.attempts));
560
+ const importedNextRetryAt = timestampField(fields, 'nextRetryAt', 'next_retry_at');
561
+ const nextRetryAt = importedNextRetryAt === undefined ? current.nextRetryAt : importedNextRetryAt;
562
+ const importedTtlAt = timestampField(fields, 'ttlAt', 'expires_at');
563
+ const ttlAt = importedTtlAt === undefined ? current.ttlAt : importedTtlAt;
564
+ const updatedAt = finiteNumber(fields['updatedAt'])
565
+ ?? finiteNumber(fields['updated_at'])
566
+ ?? finiteNumber(fields['synced_at'])
567
+ ?? current.updatedAt;
568
+ const payload = Object.prototype.hasOwnProperty.call(fields, 'payload')
569
+ ? sanitizePayload(fields['payload'])
570
+ : current.payload;
571
+ const priority = Object.prototype.hasOwnProperty.call(fields, 'priority')
572
+ ? normalizePriority(fields['priority'])
573
+ : current.priority;
574
+ const importedLastError = nullableStringField(fields, 'lastError', 'last_error', 'error');
575
+ const lastError = importedLastError === undefined ? current.lastError : importedLastError;
576
+ return {
577
+ ...current,
578
+ status,
579
+ payload,
580
+ priority,
581
+ attempts,
582
+ nextRetryAt,
583
+ ttlAt,
584
+ updatedAt,
585
+ lastError,
586
+ };
587
+ }
588
+ function buildMailboxPreparedImportPlan(batch, getById, getBaseline) {
589
+ const eligibleIds = new Set();
590
+ const insertedIds = new Set();
591
+ const managedIds = new Set();
592
+ const protectedIds = new Set();
593
+ const updatedIds = new Set();
594
+ const finalById = new Map();
595
+ for (const record of batch.records) {
596
+ if (record.kind === 'update') {
597
+ if (!eligibleIds.has(record.id))
598
+ continue;
599
+ const current = finalById.get(record.id);
600
+ if (!current)
601
+ continue;
602
+ const updated = applyImportedV1UpdateToEnvelope(current, record.fields);
603
+ if (JSON.stringify(updated) !== JSON.stringify(current))
604
+ updatedIds.add(record.id);
605
+ finalById.set(record.id, updated);
606
+ continue;
607
+ }
608
+ const envelope = { ...record.envelope };
609
+ if (eligibleIds.has(envelope.id))
610
+ continue;
611
+ let current = finalById.get(envelope.id);
612
+ if (!current) {
613
+ current = getById(envelope.id);
614
+ if (current)
615
+ finalById.set(envelope.id, current);
616
+ }
617
+ if (!current) {
618
+ finalById.set(envelope.id, envelope);
619
+ insertedIds.add(envelope.id);
620
+ eligibleIds.add(envelope.id);
621
+ protectedIds.delete(envelope.id);
622
+ continue;
623
+ }
624
+ const baseline = getBaseline(envelope.id);
625
+ if (baseline !== undefined && JSON.stringify(current) === baseline) {
626
+ eligibleIds.add(envelope.id);
627
+ managedIds.add(envelope.id);
628
+ protectedIds.delete(envelope.id);
629
+ }
630
+ else {
631
+ protectedIds.add(envelope.id);
632
+ }
633
+ }
634
+ return {
635
+ preview: Object.freeze({
636
+ sourceRecords: batch.sourceRecords,
637
+ messageCandidates: batch.messageCandidates,
638
+ uniqueMessageCandidates: batch.uniqueMessageCandidates,
639
+ updateCandidates: batch.updateCandidates,
640
+ insertedMessages: insertedIds.size,
641
+ updatedMessages: updatedIds.size,
642
+ managedMessages: managedIds.size,
643
+ protectedMessages: protectedIds.size,
644
+ }),
645
+ finalById,
646
+ insertedIds,
647
+ managedIds,
648
+ eligibleIds,
649
+ };
650
+ }
651
+ function previewPreparedMailboxDatabase(batch, database) {
652
+ try {
653
+ database.exec('BEGIN');
654
+ const hasMessages = database.prepare("SELECT 1 present FROM sqlite_master WHERE type='table' AND name='messages'").get() !== undefined;
655
+ const hasImportState = database.prepare("SELECT 1 present FROM sqlite_master WHERE type='table' AND name='v1_import_state'").get() !== undefined;
656
+ const readMessage = hasMessages
657
+ ? database.prepare('SELECT * FROM messages WHERE id=?')
658
+ : undefined;
659
+ const readBaseline = hasImportState
660
+ ? database.prepare('SELECT snapshot FROM v1_import_state WHERE id=?')
661
+ : undefined;
662
+ const plan = buildMailboxPreparedImportPlan(batch, (id) => {
663
+ const row = readMessage?.get(id);
664
+ return row ? rowToEnvelope(row) : undefined;
665
+ }, (id) => {
666
+ const snapshot = readBaseline?.get(id)?.['snapshot'];
667
+ return typeof snapshot === 'string' ? snapshot : undefined;
668
+ });
669
+ database.exec('COMMIT');
670
+ return plan.preview;
671
+ }
672
+ catch (error) {
673
+ try {
674
+ database.exec('ROLLBACK');
675
+ }
676
+ catch {
677
+ // Preserve the primary preview error.
678
+ }
679
+ throw error;
680
+ }
681
+ }
682
+ /** Preview a prepared import against an optional existing mailbox without creating or migrating it. */
683
+ export function previewPreparedMailboxImport(batch, mailboxPath) {
684
+ if (!mailboxPath) {
685
+ return buildMailboxPreparedImportPlan(batch, () => undefined, () => undefined).preview;
686
+ }
687
+ const databaseStat = mailboxPreviewStat(mailboxPath);
688
+ if (!databaseStat) {
689
+ assertMailboxPreviewTargetAbsent(mailboxPath);
690
+ return buildMailboxPreparedImportPlan(batch, () => undefined, () => undefined).preview;
691
+ }
692
+ const snapshot = stableMailboxPreviewCopy(mailboxPath);
693
+ let database;
694
+ let preview;
695
+ let operationError;
696
+ try {
697
+ database = openSqliteDatabase(snapshot.path, { readOnly: true });
698
+ preview = previewPreparedMailboxDatabase(batch, database);
699
+ }
700
+ catch (error) {
701
+ operationError = error;
702
+ }
703
+ try {
704
+ database?.close();
705
+ }
706
+ catch (error) {
707
+ operationError ??= error;
708
+ }
709
+ let cleanupError;
710
+ try {
711
+ rmSync(snapshot.dir, { recursive: true, force: true });
712
+ }
713
+ catch (error) {
714
+ cleanupError = error;
715
+ }
716
+ if (operationError !== undefined)
717
+ throw operationError;
718
+ if (cleanupError !== undefined)
719
+ throw mailboxPreviewError('cleanup_failed');
720
+ return preview;
721
+ }
252
722
  function ensureMessageColumn(db, name, definition) {
253
723
  const columns = db.prepare('PRAGMA table_info(messages)').all();
254
724
  if (!columns.some((column) => column['name'] === name)) {
@@ -671,79 +1141,74 @@ export class MailboxStore {
671
1141
  const row = this.db.prepare('SELECT workerId FROM messages WHERE id=?').get(id);
672
1142
  return typeof row?.['workerId'] === 'string' ? row['workerId'] : undefined;
673
1143
  }
674
- /** v1 messages.jsonl sqlite 迁移(幂等可重入). */
675
- importJsonl(source) {
676
- let n = 0;
677
- const eligibleIds = new Set();
1144
+ /** Build the target-aware plan shared by preview and apply. */
1145
+ buildPreparedImportPlan(batch) {
678
1146
  const readImportState = this.db.prepare('SELECT snapshot FROM v1_import_state WHERE id=?');
679
- const writeImportState = this.db.prepare('INSERT INTO v1_import_state (id,snapshot) VALUES (?,?) ON CONFLICT(id) DO UPDATE SET snapshot=excluded.snapshot');
1147
+ return buildMailboxPreparedImportPlan(batch, (id) => this.getById(id), (id) => {
1148
+ const snapshot = readImportState.get(id)?.['snapshot'];
1149
+ return typeof snapshot === 'string' ? snapshot : undefined;
1150
+ });
1151
+ }
1152
+ /** Preview the exact import result against the current mailbox snapshot without writing to it. */
1153
+ previewPreparedImport(batch) {
1154
+ this.db.exec('BEGIN');
1155
+ try {
1156
+ const preview = this.buildPreparedImportPlan(batch).preview;
1157
+ this.db.exec('COMMIT');
1158
+ return preview;
1159
+ }
1160
+ catch (error) {
1161
+ try {
1162
+ this.db.exec('ROLLBACK');
1163
+ }
1164
+ catch {
1165
+ // Preserve the primary preview error.
1166
+ }
1167
+ throw error;
1168
+ }
1169
+ }
1170
+ writeImportedEnvelopeFields(current) {
1171
+ this.db.prepare(`UPDATE messages
1172
+ SET status=?, payload=?, priority=?, attempts=?, nextRetryAt=?, ttlAt=?, updatedAt=?, lastError=?
1173
+ WHERE id=?`).run(current.status, JSON.stringify(current.payload), current.priority, current.attempts, current.nextRetryAt, current.ttlAt, current.updatedAt, current.lastError, current.id);
1174
+ }
1175
+ /** Apply a parsed batch; planning and writes share one transaction and one normalized journal. */
1176
+ importPrepared(batch) {
680
1177
  this.db.exec('BEGIN IMMEDIATE');
681
1178
  try {
682
- forEachJsonlRecord(source, (record) => {
683
- const row = asRow(record);
684
- if (!row)
685
- return;
686
- if (row['_op'] === 'update') {
687
- const id = nonEmptyString(row['id']);
688
- const fields = asRow(row['fields']);
689
- if (id && fields && eligibleIds.has(id))
690
- this.applyImportedV1Update(id, fields);
691
- return;
1179
+ const plan = this.buildPreparedImportPlan(batch);
1180
+ for (const id of plan.insertedIds) {
1181
+ const envelope = plan.finalById.get(id);
1182
+ if (!envelope || !this.send(envelope).stored) {
1183
+ throw new Error(`Mailbox import plan became stale for ${id}`);
692
1184
  }
693
- const envelope = normalizeImportedEnvelope(row);
694
- if (!envelope)
695
- return;
696
- const inserted = this.send(envelope).stored;
697
- if (inserted) {
698
- n += 1;
699
- eligibleIds.add(envelope.id);
700
- return;
701
- }
702
- const baseline = readImportState.get(envelope.id)?.['snapshot'];
703
- const current = this.getById(envelope.id);
704
- if (typeof baseline === 'string' && current && JSON.stringify(current) === baseline) {
705
- eligibleIds.add(envelope.id);
706
- }
707
- });
708
- for (const id of eligibleIds) {
709
- const current = this.getById(id);
710
- if (current)
711
- writeImportState.run(id, JSON.stringify(current));
1185
+ }
1186
+ for (const id of plan.managedIds) {
1187
+ const envelope = plan.finalById.get(id);
1188
+ if (envelope)
1189
+ this.writeImportedEnvelopeFields(envelope);
1190
+ }
1191
+ const writeImportState = this.db.prepare('INSERT INTO v1_import_state (id,snapshot) VALUES (?,?) ON CONFLICT(id) DO UPDATE SET snapshot=excluded.snapshot');
1192
+ for (const id of plan.eligibleIds) {
1193
+ const envelope = plan.finalById.get(id);
1194
+ if (envelope)
1195
+ writeImportState.run(id, JSON.stringify(envelope));
712
1196
  }
713
1197
  this.db.exec('COMMIT');
714
- return n;
1198
+ return plan.preview.insertedMessages;
715
1199
  }
716
1200
  catch (error) {
717
- this.db.exec('ROLLBACK');
1201
+ try {
1202
+ this.db.exec('ROLLBACK');
1203
+ }
1204
+ catch {
1205
+ // Preserve the primary import error.
1206
+ }
718
1207
  throw error;
719
1208
  }
720
1209
  }
721
- applyImportedV1Update(id, fields) {
722
- const current = this.getById(id);
723
- if (!current)
724
- return;
725
- const requestedStatus = importedStatus(fields['status']) ?? current.status;
726
- const status = recoverImportedStatus(current.type, requestedStatus);
727
- const attempts = Math.max(0, Math.floor(finiteNumber(fields['attempts']) ?? finiteNumber(fields['retry_count']) ?? current.attempts));
728
- const importedNextRetryAt = timestampField(fields, 'nextRetryAt', 'next_retry_at');
729
- const nextRetryAt = importedNextRetryAt === undefined ? current.nextRetryAt : importedNextRetryAt;
730
- const importedTtlAt = timestampField(fields, 'ttlAt', 'expires_at');
731
- const ttlAt = importedTtlAt === undefined ? current.ttlAt : importedTtlAt;
732
- const updatedAt = finiteNumber(fields['updatedAt'])
733
- ?? finiteNumber(fields['updated_at'])
734
- ?? finiteNumber(fields['synced_at'])
735
- ?? current.updatedAt;
736
- const payload = Object.prototype.hasOwnProperty.call(fields, 'payload')
737
- ? sanitizePayload(fields['payload'])
738
- : current.payload;
739
- const priority = Object.prototype.hasOwnProperty.call(fields, 'priority')
740
- ? normalizePriority(fields['priority'])
741
- : current.priority;
742
- const importedLastError = nullableStringField(fields, 'lastError', 'last_error', 'error');
743
- const lastError = importedLastError === undefined ? current.lastError : importedLastError;
744
- this.db.prepare(`UPDATE messages
745
- SET status=?, payload=?, priority=?, attempts=?, nextRetryAt=?, ttlAt=?, updatedAt=?, lastError=?
746
- WHERE id=?`).run(status, JSON.stringify(payload), priority, attempts, nextRetryAt, ttlAt, updatedAt, lastError, id);
1210
+ importJsonl(source) {
1211
+ return this.importPrepared(prepareMailboxJsonlImport(source));
747
1212
  }
748
1213
  close() { this.db.close(); }
749
1214
  }