@evomap/evolver-core 2.0.0-beta.8 → 2.0.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/algo/candidateAssembly.d.ts +7 -2
- package/dist/algo/candidateAssembly.js +81 -13
- package/dist/algo/capabilityCandidates.js +10 -0
- package/dist/algo/conversationSniffer.js +20 -7
- package/dist/algo/cycleEngine.d.ts +23 -2
- package/dist/algo/cycleEngine.js +236 -36
- package/dist/algo/exploration.d.ts +7 -0
- package/dist/algo/exploration.js +16 -3
- package/dist/algo/geneHealth.d.ts +2 -2
- package/dist/algo/geneHealth.js +5 -4
- package/dist/algo/geneSelection.d.ts +63 -5
- package/dist/algo/geneSelection.js +330 -30
- package/dist/algo/index.d.ts +2 -0
- package/dist/algo/index.js +2 -0
- package/dist/algo/orchestrator.d.ts +8 -1
- package/dist/algo/orchestrator.js +15 -3
- package/dist/algo/publishEligibility.d.ts +34 -0
- package/dist/algo/publishEligibility.js +52 -0
- package/dist/algo/ucb1.d.ts +53 -0
- package/dist/algo/ucb1.js +156 -0
- package/dist/assetstore/assetSidecarRecords.d.ts +7 -2
- package/dist/assetstore/assetSidecarRecords.js +238 -0
- package/dist/assetstore/assetSidecarRecovery.js +2 -2
- package/dist/assetstore/assetStoreHealth.d.ts +6 -0
- package/dist/assetstore/assetStoreHealth.js +56 -14
- package/dist/assetstore/assetStoreStorage.d.ts +1 -1
- package/dist/assetstore/assetStoreStorage.js +27 -9
- package/dist/assetstore/assetSyncLedger.d.ts +86 -0
- package/dist/assetstore/assetSyncLedger.js +683 -4
- package/dist/assetstore/index.d.ts +1 -0
- package/dist/assetstore/index.js +1 -0
- package/dist/assetstore/localAssetStoreSnapshot.d.ts +51 -0
- package/dist/assetstore/localAssetStoreSnapshot.js +329 -0
- package/dist/assetstore/localJsonl.d.ts +4 -2
- package/dist/assetstore/localJsonl.js +91 -11
- package/dist/assetstore/provenance.d.ts +80 -4
- package/dist/assetstore/provenance.js +313 -5
- package/dist/assetstore/provider.d.ts +58 -1
- package/dist/assetstore/provider.js +97 -6
- package/dist/benchmark/antiGeneBenchmark.d.ts +2 -0
- package/dist/benchmark/antiGeneBenchmark.js +4 -3
- package/dist/benchmark/antiGeneRollout.d.ts +2 -0
- package/dist/benchmark/antiGeneRollout.js +4 -3
- package/dist/benchmark/index.d.ts +2 -1
- package/dist/benchmark/index.js +2 -1
- package/dist/benchmark/selectionFlatAbstention.d.ts +152 -0
- package/dist/benchmark/selectionFlatAbstention.js +481 -0
- package/dist/bootstrap/envFingerprint.d.ts +9 -0
- package/dist/bootstrap/envFingerprint.js +5 -0
- package/dist/bootstrap/index.d.ts +2 -1
- package/dist/bootstrap/index.js +2 -1
- package/dist/bootstrap/v1EnvCompat.d.ts +111 -0
- package/dist/bootstrap/v1EnvCompat.js +280 -0
- package/dist/events/eventArchive.d.ts +2 -0
- package/dist/events/eventArchive.js +13 -3
- package/dist/events/eventSchema.d.ts +7 -7
- package/dist/events/eventStore.d.ts +2 -0
- package/dist/events/eventStore.js +5 -1
- package/dist/events/ingest.d.ts +1 -0
- package/dist/events/ingest.js +1 -0
- package/dist/events/paths.d.ts +3 -1
- package/dist/events/paths.js +4 -0
- package/dist/events/public.d.ts +2 -2
- package/dist/events/public.js +2 -2
- package/dist/events/reports.d.ts +2 -0
- package/dist/events/reports.js +4 -0
- package/dist/exec/autoExec.d.ts +43 -2
- package/dist/exec/autoExec.js +76 -9
- package/dist/exec/autonomousCycle.d.ts +22 -4
- package/dist/exec/autonomousCycle.js +64 -13
- package/dist/exec/claudeBridge.d.ts +32 -7
- package/dist/exec/claudeBridge.js +281 -29
- package/dist/exec/prompt.js +5 -1
- package/dist/exec/runnerRegistry.d.ts +68 -26
- package/dist/exec/runnerRegistry.js +307 -72
- package/dist/exec/selfPr.js +1 -7
- package/dist/feedback/envelope.d.ts +61 -0
- package/dist/feedback/envelope.js +168 -0
- package/dist/feedback/index.d.ts +1 -0
- package/dist/feedback/index.js +1 -0
- package/dist/hooks/hooks.js +1 -0
- package/dist/hub/assetCallLog.d.ts +35 -1
- package/dist/hub/assetCallLog.js +124 -1
- package/dist/hub/bindings.d.ts +8 -1
- package/dist/hub/bindings.js +17 -6
- package/dist/hub/capability.d.ts +90 -3
- package/dist/hub/fake.d.ts +2 -2
- package/dist/hub/fake.js +1 -1
- package/dist/hub/questionGenerator.d.ts +5 -1
- package/dist/hub/questionGenerator.js +8 -6
- package/dist/hub/sanitize.js +3 -2
- package/dist/index.d.ts +3 -1
- package/dist/index.js +4 -1
- package/dist/mailbox/dispatch.d.ts +1 -1
- package/dist/mailbox/dispatch.js +22 -6
- package/dist/mailbox/envelope.d.ts +7 -1
- package/dist/mailbox/envelope.js +9 -2
- package/dist/mailbox/ipcServer.d.ts +10 -2
- package/dist/mailbox/ipcServer.js +163 -13
- package/dist/mailbox/store.d.ts +89 -3
- package/dist/mailbox/store.js +895 -41
- package/dist/ops/evolutionGraphProjection.d.ts +20 -0
- package/dist/ops/evolutionGraphProjection.js +315 -0
- package/dist/ops/index.d.ts +2 -1
- package/dist/ops/index.js +2 -1
- package/dist/ops/selfUpdate.d.ts +8 -0
- package/dist/ops/selfUpdate.js +24 -8
- package/dist/personality/schema.d.ts +16 -16
- package/dist/schema/evolutionGraph.d.ts +784 -0
- package/dist/schema/evolutionGraph.js +187 -0
- package/dist/schema/index.d.ts +1 -0
- package/dist/schema/index.js +1 -0
- package/dist/schema/signal.d.ts +3 -3
- package/dist/schema/signal.js +1 -1
- package/dist/signals/curriculum.d.ts +55 -0
- package/dist/signals/curriculum.js +202 -0
- package/dist/signals/cycleHistoryFromEvents.js +17 -8
- package/dist/signals/expand.d.ts +15 -1
- package/dist/signals/expand.js +169 -1
- package/dist/signals/extractor.d.ts +2 -2
- package/dist/signals/extractor.js +31 -6
- package/dist/signals/index.d.ts +3 -1
- package/dist/signals/index.js +3 -1
- package/dist/signals/metaSignals.d.ts +4 -0
- package/dist/signals/metaSignals.js +42 -0
- package/dist/signals/signalGate.js +1 -1
- package/dist/signals/taskDomain.d.ts +22 -0
- package/dist/signals/taskDomain.js +43 -0
- package/dist/strategy/constraintAblation.d.ts +64 -0
- package/dist/strategy/constraintAblation.js +2820 -0
- package/dist/strategy/constraintAblationPredicates.d.ts +31 -0
- package/dist/strategy/constraintAblationPredicates.js +339 -0
- package/dist/strategy/index.d.ts +2 -1
- package/dist/strategy/index.js +2 -1
- package/dist/trace/index.d.ts +3 -1
- package/dist/trace/index.js +3 -1
- package/dist/trace/learningTrace.d.ts +209 -0
- package/dist/trace/learningTrace.js +276 -0
- package/dist/trace/proxyTurns.d.ts +31 -0
- package/dist/trace/proxyTurns.js +137 -0
- package/dist/verify/sandboxRunner.d.ts +28 -0
- package/dist/verify/sandboxRunner.js +218 -19
- package/dist/verify/sandboxedValidation.d.ts +9 -0
- package/dist/verify/sandboxedValidation.js +113 -13
- package/dist/verify/validation.d.ts +11 -1
- package/dist/verify/validation.js +31 -0
- package/package.json +5 -2
package/dist/assetstore/index.js
CHANGED
|
@@ -0,0 +1,51 @@
|
|
|
1
|
+
import { type AssetRecord } from './provider.js';
|
|
2
|
+
import type { ProvenanceRecord } from './provenance.js';
|
|
3
|
+
export type CorruptLocalAssetStoreReason = 'invalid_utf8' | 'unterminated' | 'invalid_json' | 'invalid_record';
|
|
4
|
+
export declare class CorruptLocalAssetStoreError extends Error {
|
|
5
|
+
readonly reason: CorruptLocalAssetStoreReason;
|
|
6
|
+
readonly file?: string | undefined;
|
|
7
|
+
readonly row?: number | undefined;
|
|
8
|
+
readonly code = "CORRUPT_LOCAL_ASSET_STORE";
|
|
9
|
+
constructor(reason: CorruptLocalAssetStoreReason, file?: string | undefined, row?: number | undefined);
|
|
10
|
+
}
|
|
11
|
+
export type LocalAssetStoreSnapshotLimitReason = 'file_bytes' | 'total_bytes' | 'records';
|
|
12
|
+
export declare class LocalAssetStoreSnapshotLimitError extends Error {
|
|
13
|
+
readonly reason: LocalAssetStoreSnapshotLimitReason;
|
|
14
|
+
readonly file?: string | undefined;
|
|
15
|
+
readonly code = "LOCAL_ASSET_STORE_SNAPSHOT_LIMIT";
|
|
16
|
+
constructor(reason: LocalAssetStoreSnapshotLimitReason, file?: string | undefined);
|
|
17
|
+
}
|
|
18
|
+
export type LocalAssetStoreSnapshotChangeReason = 'changed' | 'lock_present' | 'invalid_snapshot';
|
|
19
|
+
export declare class LocalAssetStoreSnapshotChangedError extends Error {
|
|
20
|
+
readonly reason: LocalAssetStoreSnapshotChangeReason;
|
|
21
|
+
readonly code = "LOCAL_ASSET_STORE_SNAPSHOT_CHANGED";
|
|
22
|
+
constructor(reason: LocalAssetStoreSnapshotChangeReason);
|
|
23
|
+
}
|
|
24
|
+
export interface ReadLocalAssetStoreSnapshotOptions {
|
|
25
|
+
maxFileBytes?: number;
|
|
26
|
+
maxTotalBytes?: number;
|
|
27
|
+
maxRecords?: number;
|
|
28
|
+
}
|
|
29
|
+
export interface ReadLocalAssetStoreSnapshotDeps {
|
|
30
|
+
/** Test seam invoked after every tracked file has been read but before the stable-read verification. */
|
|
31
|
+
beforeVerify?: () => void;
|
|
32
|
+
}
|
|
33
|
+
export interface LocalAssetStoreSnapshot {
|
|
34
|
+
readonly exists: boolean;
|
|
35
|
+
readonly fingerprint: string;
|
|
36
|
+
readonly assets: ReadonlyMap<string, AssetRecord>;
|
|
37
|
+
readonly provenance: ReadonlyMap<string, ProvenanceRecord>;
|
|
38
|
+
readonly stats: {
|
|
39
|
+
readonly bytes: number;
|
|
40
|
+
readonly assetRows: number;
|
|
41
|
+
readonly provenanceRows: number;
|
|
42
|
+
readonly uniqueAssets: number;
|
|
43
|
+
};
|
|
44
|
+
}
|
|
45
|
+
/**
|
|
46
|
+
* Read a bounded, strict target-store snapshot without creating the directory, a lock, or any other file.
|
|
47
|
+
* Writers are detected through the lock/root/file fingerprints; callers must revalidate before apply.
|
|
48
|
+
*/
|
|
49
|
+
export declare function readLocalAssetStoreSnapshot(baseDir: string, options?: ReadLocalAssetStoreSnapshotOptions, deps?: ReadLocalAssetStoreSnapshotDeps): LocalAssetStoreSnapshot;
|
|
50
|
+
/** Fail closed when the target store changed between planning and apply. Performs reads only. */
|
|
51
|
+
export declare function assertLocalAssetStoreSnapshotCurrent(snapshot: LocalAssetStoreSnapshot): void;
|
|
@@ -0,0 +1,329 @@
|
|
|
1
|
+
import { createHash } from 'node:crypto';
|
|
2
|
+
import { lstatSync } from 'node:fs';
|
|
3
|
+
import { join, resolve } from 'node:path';
|
|
4
|
+
import { LOCAL_ASSET_FILES } from './assetStoreLayout.js';
|
|
5
|
+
import { AssetStoreReadLimitError, readRegularBuffer, UnsafeAssetStorePathError, } from './assetStoreStorage.js';
|
|
6
|
+
import { parseProvenanceRecord, CorruptAssetSidecarError, } from './assetSidecarRecords.js';
|
|
7
|
+
import { frozenAssetRecordsEqual, FrozenAssetIdCollisionError } from './provider.js';
|
|
8
|
+
const PROVENANCE_FILE = 'provenance.jsonl';
|
|
9
|
+
const LOCK_FILE = '.assetstore.lock';
|
|
10
|
+
const DEFAULT_MAX_FILE_BYTES = 64 * 1024 * 1024;
|
|
11
|
+
const DEFAULT_MAX_TOTAL_BYTES = 128 * 1024 * 1024;
|
|
12
|
+
const DEFAULT_MAX_RECORDS = 100_000;
|
|
13
|
+
export class CorruptLocalAssetStoreError extends Error {
|
|
14
|
+
reason;
|
|
15
|
+
file;
|
|
16
|
+
row;
|
|
17
|
+
code = 'CORRUPT_LOCAL_ASSET_STORE';
|
|
18
|
+
constructor(reason, file, row) {
|
|
19
|
+
super(`corrupt local asset store: ${reason}${file ? `/${file}` : ''}${row ? `/${row}` : ''}`);
|
|
20
|
+
this.reason = reason;
|
|
21
|
+
this.file = file;
|
|
22
|
+
this.row = row;
|
|
23
|
+
this.name = 'CorruptLocalAssetStoreError';
|
|
24
|
+
}
|
|
25
|
+
}
|
|
26
|
+
export class LocalAssetStoreSnapshotLimitError extends Error {
|
|
27
|
+
reason;
|
|
28
|
+
file;
|
|
29
|
+
code = 'LOCAL_ASSET_STORE_SNAPSHOT_LIMIT';
|
|
30
|
+
constructor(reason, file) {
|
|
31
|
+
super(`local asset store snapshot limit exceeded: ${reason}${file ? `/${file}` : ''}`);
|
|
32
|
+
this.reason = reason;
|
|
33
|
+
this.file = file;
|
|
34
|
+
this.name = 'LocalAssetStoreSnapshotLimitError';
|
|
35
|
+
}
|
|
36
|
+
}
|
|
37
|
+
export class LocalAssetStoreSnapshotChangedError extends Error {
|
|
38
|
+
reason;
|
|
39
|
+
code = 'LOCAL_ASSET_STORE_SNAPSHOT_CHANGED';
|
|
40
|
+
constructor(reason) {
|
|
41
|
+
super(`local asset store snapshot changed: ${reason}`);
|
|
42
|
+
this.reason = reason;
|
|
43
|
+
this.name = 'LocalAssetStoreSnapshotChangedError';
|
|
44
|
+
}
|
|
45
|
+
}
|
|
46
|
+
class SnapshotMap {
|
|
47
|
+
#values;
|
|
48
|
+
constructor(entries) {
|
|
49
|
+
this.#values = new Map(entries);
|
|
50
|
+
Object.freeze(this);
|
|
51
|
+
}
|
|
52
|
+
get size() { return this.#values.size; }
|
|
53
|
+
get(key) { return this.#values.get(key); }
|
|
54
|
+
has(key) { return this.#values.has(key); }
|
|
55
|
+
entries() { return this.#values.entries(); }
|
|
56
|
+
keys() { return this.#values.keys(); }
|
|
57
|
+
values() { return this.#values.values(); }
|
|
58
|
+
[Symbol.iterator]() { return this.#values[Symbol.iterator](); }
|
|
59
|
+
forEach(callbackfn, thisArg) {
|
|
60
|
+
for (const [key, value] of this.#values)
|
|
61
|
+
callbackfn.call(thisArg, value, key, this);
|
|
62
|
+
}
|
|
63
|
+
}
|
|
64
|
+
const metadataBySnapshot = new WeakMap();
|
|
65
|
+
function isErrno(error, code) {
|
|
66
|
+
return typeof error === 'object' && error !== null && error.code === code;
|
|
67
|
+
}
|
|
68
|
+
function boundedPositiveInt(value, fallback) {
|
|
69
|
+
return value === undefined || !Number.isFinite(value)
|
|
70
|
+
? fallback
|
|
71
|
+
: Math.max(1, Math.floor(value));
|
|
72
|
+
}
|
|
73
|
+
function statFingerprint(stat) {
|
|
74
|
+
return `${stat.dev}:${stat.ino}:${stat.mode}:${stat.size}:${stat.mtimeNs}:${stat.ctimeNs}`;
|
|
75
|
+
}
|
|
76
|
+
function directoryFingerprint(path) {
|
|
77
|
+
try {
|
|
78
|
+
const stat = lstatSync(path, { bigint: true });
|
|
79
|
+
if (stat.isSymbolicLink())
|
|
80
|
+
throw new UnsafeAssetStorePathError('base_directory', 'symlink');
|
|
81
|
+
if (!stat.isDirectory())
|
|
82
|
+
throw new UnsafeAssetStorePathError('base_directory', 'not_directory');
|
|
83
|
+
return statFingerprint(stat);
|
|
84
|
+
}
|
|
85
|
+
catch (error) {
|
|
86
|
+
if (isErrno(error, 'ENOENT'))
|
|
87
|
+
return 'missing';
|
|
88
|
+
throw error;
|
|
89
|
+
}
|
|
90
|
+
}
|
|
91
|
+
function optionalFileFingerprint(path, role) {
|
|
92
|
+
try {
|
|
93
|
+
const stat = lstatSync(path, { bigint: true });
|
|
94
|
+
if (stat.isSymbolicLink())
|
|
95
|
+
throw new UnsafeAssetStorePathError(role, 'symlink');
|
|
96
|
+
if (!stat.isFile())
|
|
97
|
+
throw new UnsafeAssetStorePathError(role, 'not_regular_file');
|
|
98
|
+
return statFingerprint(stat);
|
|
99
|
+
}
|
|
100
|
+
catch (error) {
|
|
101
|
+
if (isErrno(error, 'ENOENT'))
|
|
102
|
+
return 'missing';
|
|
103
|
+
throw error;
|
|
104
|
+
}
|
|
105
|
+
}
|
|
106
|
+
function captureState(baseDir) {
|
|
107
|
+
const root = directoryFingerprint(baseDir);
|
|
108
|
+
const files = new Map();
|
|
109
|
+
if (root === 'missing') {
|
|
110
|
+
for (const kind of Object.keys(LOCAL_ASSET_FILES))
|
|
111
|
+
files.set(kind, 'missing');
|
|
112
|
+
files.set('provenance', 'missing');
|
|
113
|
+
return { root, lock: 'missing', files };
|
|
114
|
+
}
|
|
115
|
+
for (const [kind, file] of Object.entries(LOCAL_ASSET_FILES)) {
|
|
116
|
+
files.set(kind, optionalFileFingerprint(join(baseDir, file), 'asset_file'));
|
|
117
|
+
}
|
|
118
|
+
files.set('provenance', optionalFileFingerprint(join(baseDir, PROVENANCE_FILE), 'asset_file'));
|
|
119
|
+
return {
|
|
120
|
+
root,
|
|
121
|
+
lock: optionalFileFingerprint(join(baseDir, LOCK_FILE), 'lock_file'),
|
|
122
|
+
files,
|
|
123
|
+
};
|
|
124
|
+
}
|
|
125
|
+
function recaptureState(baseDir) {
|
|
126
|
+
try {
|
|
127
|
+
return captureState(baseDir);
|
|
128
|
+
}
|
|
129
|
+
catch (error) {
|
|
130
|
+
if (error instanceof UnsafeAssetStorePathError) {
|
|
131
|
+
throw new LocalAssetStoreSnapshotChangedError('changed');
|
|
132
|
+
}
|
|
133
|
+
throw error;
|
|
134
|
+
}
|
|
135
|
+
}
|
|
136
|
+
function statesEqual(left, right) {
|
|
137
|
+
if (left.root !== right.root || left.lock !== right.lock || left.files.size !== right.files.size)
|
|
138
|
+
return false;
|
|
139
|
+
for (const [file, fingerprint] of left.files) {
|
|
140
|
+
if (right.files.get(file) !== fingerprint)
|
|
141
|
+
return false;
|
|
142
|
+
}
|
|
143
|
+
return true;
|
|
144
|
+
}
|
|
145
|
+
function snapshotFingerprint(baseDir, state) {
|
|
146
|
+
const fileState = [...state.files.entries()];
|
|
147
|
+
const digest = createHash('sha256')
|
|
148
|
+
.update(JSON.stringify([baseDir, state.root, state.lock, fileState]))
|
|
149
|
+
.digest('hex');
|
|
150
|
+
return `sha256:${digest}`;
|
|
151
|
+
}
|
|
152
|
+
function decodeStrict(bytes, file) {
|
|
153
|
+
try {
|
|
154
|
+
return new TextDecoder('utf-8', { fatal: true }).decode(bytes);
|
|
155
|
+
}
|
|
156
|
+
catch {
|
|
157
|
+
throw new CorruptLocalAssetStoreError('invalid_utf8', file);
|
|
158
|
+
}
|
|
159
|
+
}
|
|
160
|
+
function assertTerminated(raw, file) {
|
|
161
|
+
if (raw.length > 0 && !raw.endsWith('\n'))
|
|
162
|
+
throw new CorruptLocalAssetStoreError('unterminated', file);
|
|
163
|
+
}
|
|
164
|
+
function immutableJson(value) {
|
|
165
|
+
if (!value || typeof value !== 'object')
|
|
166
|
+
return value;
|
|
167
|
+
const pending = [value];
|
|
168
|
+
const seen = new Set();
|
|
169
|
+
while (pending.length > 0) {
|
|
170
|
+
const current = pending.pop();
|
|
171
|
+
if (seen.has(current))
|
|
172
|
+
continue;
|
|
173
|
+
seen.add(current);
|
|
174
|
+
for (const item of Object.values(current)) {
|
|
175
|
+
if (item && typeof item === 'object')
|
|
176
|
+
pending.push(item);
|
|
177
|
+
}
|
|
178
|
+
Object.freeze(current);
|
|
179
|
+
}
|
|
180
|
+
return value;
|
|
181
|
+
}
|
|
182
|
+
function parseAssetFile(bytes, kind, file, assets, consumeRecord) {
|
|
183
|
+
const raw = decodeStrict(bytes, file);
|
|
184
|
+
assertTerminated(raw, file);
|
|
185
|
+
let row = 0;
|
|
186
|
+
for (const line of raw.split('\n')) {
|
|
187
|
+
if (!line.trim())
|
|
188
|
+
continue;
|
|
189
|
+
row += 1;
|
|
190
|
+
consumeRecord();
|
|
191
|
+
let value;
|
|
192
|
+
try {
|
|
193
|
+
value = JSON.parse(line);
|
|
194
|
+
}
|
|
195
|
+
catch {
|
|
196
|
+
throw new CorruptLocalAssetStoreError('invalid_json', file, row);
|
|
197
|
+
}
|
|
198
|
+
if (!value || typeof value !== 'object' || Array.isArray(value)) {
|
|
199
|
+
throw new CorruptLocalAssetStoreError('invalid_record', file, row);
|
|
200
|
+
}
|
|
201
|
+
const candidate = value;
|
|
202
|
+
if (candidate['type'] !== kind
|
|
203
|
+
|| typeof candidate['asset_id'] !== 'string'
|
|
204
|
+
|| !candidate['asset_id']
|
|
205
|
+
|| candidate['asset_id'] !== candidate['asset_id'].trim()) {
|
|
206
|
+
throw new CorruptLocalAssetStoreError('invalid_record', file, row);
|
|
207
|
+
}
|
|
208
|
+
const record = candidate;
|
|
209
|
+
const existing = assets.get(record.asset_id);
|
|
210
|
+
if (existing && !frozenAssetRecordsEqual(existing, record)) {
|
|
211
|
+
throw new FrozenAssetIdCollisionError(record.asset_id);
|
|
212
|
+
}
|
|
213
|
+
if (!existing)
|
|
214
|
+
assets.set(record.asset_id, immutableJson(record));
|
|
215
|
+
}
|
|
216
|
+
return row;
|
|
217
|
+
}
|
|
218
|
+
function parseProvenanceFile(bytes, consumeRecord) {
|
|
219
|
+
const raw = decodeStrict(bytes, PROVENANCE_FILE);
|
|
220
|
+
const records = new Map();
|
|
221
|
+
let rows = 0;
|
|
222
|
+
if (raw.length > 0 && !raw.endsWith('\n')) {
|
|
223
|
+
throw new CorruptAssetSidecarError('provenance', 'unterminated');
|
|
224
|
+
}
|
|
225
|
+
for (const line of raw.split('\n')) {
|
|
226
|
+
if (!line.trim())
|
|
227
|
+
continue;
|
|
228
|
+
rows += 1;
|
|
229
|
+
consumeRecord();
|
|
230
|
+
let record;
|
|
231
|
+
try {
|
|
232
|
+
record = parseProvenanceRecord(JSON.parse(line));
|
|
233
|
+
}
|
|
234
|
+
catch {
|
|
235
|
+
record = null;
|
|
236
|
+
}
|
|
237
|
+
if (!record)
|
|
238
|
+
throw new CorruptAssetSidecarError('provenance', 'invalid_row');
|
|
239
|
+
records.set(record.assetId, immutableJson(record));
|
|
240
|
+
}
|
|
241
|
+
return { records: new SnapshotMap(records), rows };
|
|
242
|
+
}
|
|
243
|
+
function readTrackedFile(path, file, maxFileBytes) {
|
|
244
|
+
try {
|
|
245
|
+
return readRegularBuffer(path, maxFileBytes) ?? Buffer.alloc(0);
|
|
246
|
+
}
|
|
247
|
+
catch (error) {
|
|
248
|
+
if (error instanceof AssetStoreReadLimitError) {
|
|
249
|
+
throw new LocalAssetStoreSnapshotLimitError('file_bytes', file);
|
|
250
|
+
}
|
|
251
|
+
if (isErrno(error, 'ENOENT') || isErrno(error, 'ELOOP') || isErrno(error, 'EISDIR')) {
|
|
252
|
+
throw new LocalAssetStoreSnapshotChangedError('changed');
|
|
253
|
+
}
|
|
254
|
+
if (error instanceof UnsafeAssetStorePathError && error.reason === 'path_changed') {
|
|
255
|
+
throw new LocalAssetStoreSnapshotChangedError('changed');
|
|
256
|
+
}
|
|
257
|
+
throw error;
|
|
258
|
+
}
|
|
259
|
+
}
|
|
260
|
+
/**
|
|
261
|
+
* Read a bounded, strict target-store snapshot without creating the directory, a lock, or any other file.
|
|
262
|
+
* Writers are detected through the lock/root/file fingerprints; callers must revalidate before apply.
|
|
263
|
+
*/
|
|
264
|
+
export function readLocalAssetStoreSnapshot(baseDir, options = {}, deps = {}) {
|
|
265
|
+
const resolvedBaseDir = resolve(baseDir);
|
|
266
|
+
const maxFileBytes = boundedPositiveInt(options.maxFileBytes, DEFAULT_MAX_FILE_BYTES);
|
|
267
|
+
const maxTotalBytes = boundedPositiveInt(options.maxTotalBytes, DEFAULT_MAX_TOTAL_BYTES);
|
|
268
|
+
const maxRecords = boundedPositiveInt(options.maxRecords, DEFAULT_MAX_RECORDS);
|
|
269
|
+
const before = captureState(resolvedBaseDir);
|
|
270
|
+
if (before.lock !== 'missing')
|
|
271
|
+
throw new LocalAssetStoreSnapshotChangedError('lock_present');
|
|
272
|
+
const buffers = new Map();
|
|
273
|
+
let totalBytes = 0;
|
|
274
|
+
if (before.root !== 'missing') {
|
|
275
|
+
for (const [kind, file] of Object.entries(LOCAL_ASSET_FILES)) {
|
|
276
|
+
const bytes = readTrackedFile(join(resolvedBaseDir, file), file, maxFileBytes);
|
|
277
|
+
totalBytes += bytes.length;
|
|
278
|
+
if (totalBytes > maxTotalBytes)
|
|
279
|
+
throw new LocalAssetStoreSnapshotLimitError('total_bytes');
|
|
280
|
+
buffers.set(kind, bytes);
|
|
281
|
+
}
|
|
282
|
+
const provenanceBytes = readTrackedFile(join(resolvedBaseDir, PROVENANCE_FILE), PROVENANCE_FILE, maxFileBytes);
|
|
283
|
+
totalBytes += provenanceBytes.length;
|
|
284
|
+
if (totalBytes > maxTotalBytes)
|
|
285
|
+
throw new LocalAssetStoreSnapshotLimitError('total_bytes');
|
|
286
|
+
buffers.set('provenance', provenanceBytes);
|
|
287
|
+
}
|
|
288
|
+
deps.beforeVerify?.();
|
|
289
|
+
const after = recaptureState(resolvedBaseDir);
|
|
290
|
+
if (after.lock !== 'missing' || !statesEqual(before, after)) {
|
|
291
|
+
throw new LocalAssetStoreSnapshotChangedError(after.lock !== 'missing' ? 'lock_present' : 'changed');
|
|
292
|
+
}
|
|
293
|
+
let records = 0;
|
|
294
|
+
const consumeRecords = (count) => {
|
|
295
|
+
records += count;
|
|
296
|
+
if (records > maxRecords)
|
|
297
|
+
throw new LocalAssetStoreSnapshotLimitError('records');
|
|
298
|
+
};
|
|
299
|
+
const assets = new Map();
|
|
300
|
+
let assetRows = 0;
|
|
301
|
+
for (const [kind, file] of Object.entries(LOCAL_ASSET_FILES)) {
|
|
302
|
+
assetRows += parseAssetFile(buffers.get(kind) ?? Buffer.alloc(0), kind, file, assets, () => consumeRecords(1));
|
|
303
|
+
}
|
|
304
|
+
const parsedProvenance = parseProvenanceFile(buffers.get('provenance') ?? Buffer.alloc(0), () => consumeRecords(1));
|
|
305
|
+
const snapshot = Object.freeze({
|
|
306
|
+
exists: before.root !== 'missing',
|
|
307
|
+
fingerprint: snapshotFingerprint(resolvedBaseDir, before),
|
|
308
|
+
assets: new SnapshotMap(assets),
|
|
309
|
+
provenance: parsedProvenance.records,
|
|
310
|
+
stats: Object.freeze({
|
|
311
|
+
bytes: totalBytes,
|
|
312
|
+
assetRows,
|
|
313
|
+
provenanceRows: parsedProvenance.rows,
|
|
314
|
+
uniqueAssets: assets.size,
|
|
315
|
+
}),
|
|
316
|
+
});
|
|
317
|
+
metadataBySnapshot.set(snapshot, { baseDir: resolvedBaseDir, state: before });
|
|
318
|
+
return snapshot;
|
|
319
|
+
}
|
|
320
|
+
/** Fail closed when the target store changed between planning and apply. Performs reads only. */
|
|
321
|
+
export function assertLocalAssetStoreSnapshotCurrent(snapshot) {
|
|
322
|
+
const metadata = metadataBySnapshot.get(snapshot);
|
|
323
|
+
if (!metadata)
|
|
324
|
+
throw new LocalAssetStoreSnapshotChangedError('invalid_snapshot');
|
|
325
|
+
const current = recaptureState(metadata.baseDir);
|
|
326
|
+
if (current.lock !== 'missing' || !statesEqual(metadata.state, current)) {
|
|
327
|
+
throw new LocalAssetStoreSnapshotChangedError(current.lock !== 'missing' ? 'lock_present' : 'changed');
|
|
328
|
+
}
|
|
329
|
+
}
|
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
import { type AssetKind, type AssetRecord, type AssetStoreProvider, type PutResult, type SearchQuery } from './provider.js';
|
|
1
|
+
import { type AssetKind, type AssetRecord, type AssetStoreProvider, type ConditionalPutOptions, type ConditionalPutResult, type PutResult, type SearchQuery } from './provider.js';
|
|
2
2
|
/**
|
|
3
3
|
* 本地 jsonl 资产库(M3-2, 移植 v1 src/gep/assetStore.js 单写锁).
|
|
4
4
|
* 每 kind 一文件(genes/capsules/events.jsonl); append-only; O_EXCL 文件锁防并发写撕裂;
|
|
@@ -19,13 +19,15 @@ export declare class LocalJsonlProvider implements AssetStoreProvider {
|
|
|
19
19
|
private ensureFresh;
|
|
20
20
|
private updateFileStateAfterWrite;
|
|
21
21
|
put(asset: AssetRecord): Promise<PutResult>;
|
|
22
|
+
putConditional(asset: AssetRecord, options?: ConditionalPutOptions): Promise<ConditionalPutResult>;
|
|
22
23
|
/**
|
|
23
24
|
* 迁移专用(M8-2): 以**冻结 asset_id** 原样写入, 不经 normalizeForPut 重算/校验.
|
|
24
25
|
* 仅 v1→v2 导入用(硬化 A6 存量冻结); 普通写一律走 put(). record 必须自带 asset_id.
|
|
25
26
|
*/
|
|
26
27
|
putFrozen(record: AssetRecord): Promise<PutResult>;
|
|
28
|
+
putFrozenConditional(record: AssetRecord, options?: ConditionalPutOptions): Promise<ConditionalPutResult>;
|
|
27
29
|
get(assetId: string): Promise<AssetRecord | null>;
|
|
28
|
-
findByLogicalId(id: string, limit?: number): Promise<AssetRecord[]>;
|
|
30
|
+
findByLogicalId(id: string, limit?: number, kind?: AssetKind): Promise<AssetRecord[]>;
|
|
29
31
|
list(kind?: AssetKind, limit?: number): Promise<AssetRecord[]>;
|
|
30
32
|
search(q: SearchQuery): Promise<AssetRecord[]>;
|
|
31
33
|
/**
|
|
@@ -2,7 +2,12 @@ import { join } from 'node:path';
|
|
|
2
2
|
import { acquireLock, releaseLock } from '../util/fileLock.js';
|
|
3
3
|
import { appendUtf8Durable, assertAssetStoreDirectory, assertOptionalRegularFile, ensureAssetStoreDirectory, readUtf8Regular, regularFileFingerprint, replaceUtf8Durable, } from './assetStoreStorage.js';
|
|
4
4
|
import { LOCAL_ASSET_FILES } from './assetStoreLayout.js';
|
|
5
|
-
import { normalizeForPut, } from './provider.js';
|
|
5
|
+
import { FrozenAssetIdCollisionError, frozenAssetRecordsEqual, normalizeForPut, } from './provider.js';
|
|
6
|
+
function resultLogicalId(logicalId) {
|
|
7
|
+
return logicalId !== undefined && logicalId.length > 0 && logicalId === logicalId.trim()
|
|
8
|
+
? logicalId
|
|
9
|
+
: undefined;
|
|
10
|
+
}
|
|
6
11
|
function signalsOf(a) {
|
|
7
12
|
const out = [];
|
|
8
13
|
for (const key of ['signals_match', 'signals', 'trigger', 'trigger_signals']) {
|
|
@@ -97,15 +102,45 @@ export class LocalJsonlProvider {
|
|
|
97
102
|
this.loaded = true;
|
|
98
103
|
}
|
|
99
104
|
async put(asset) {
|
|
105
|
+
return this.putConditional(asset, { allowLogicalCollision: true });
|
|
106
|
+
}
|
|
107
|
+
async putConditional(asset, options) {
|
|
100
108
|
const { record, verified } = normalizeForPut(asset);
|
|
101
109
|
const file = join(this.baseDir, LOCAL_ASSET_FILES[record.type]);
|
|
110
|
+
const logicalId = typeof record.id === 'string' ? record.id : undefined;
|
|
111
|
+
let collision;
|
|
102
112
|
assertOptionalRegularFile(this.lockPath, 'lock_file');
|
|
103
113
|
acquireLock(this.lockPath);
|
|
104
114
|
try {
|
|
105
115
|
// Refresh under the shared lock so another process cannot append between reload and dedupe.
|
|
106
116
|
this.refreshUnderLock();
|
|
107
|
-
|
|
108
|
-
|
|
117
|
+
const existing = this.index.get(record.asset_id);
|
|
118
|
+
if (existing) {
|
|
119
|
+
if (!frozenAssetRecordsEqual(existing, record))
|
|
120
|
+
throw new FrozenAssetIdCollisionError(record.asset_id);
|
|
121
|
+
return {
|
|
122
|
+
asset_id: record.asset_id,
|
|
123
|
+
stored: false,
|
|
124
|
+
verified,
|
|
125
|
+
status: 'already_exists',
|
|
126
|
+
...(resultLogicalId(logicalId) ? { logicalId } : {}),
|
|
127
|
+
};
|
|
128
|
+
}
|
|
129
|
+
collision = logicalId === undefined
|
|
130
|
+
? undefined
|
|
131
|
+
: [...this.index.values()].find((existing) => (existing.type === record.type
|
|
132
|
+
&& existing.id === logicalId
|
|
133
|
+
&& existing.asset_id !== record.asset_id));
|
|
134
|
+
if (collision && !options?.allowLogicalCollision) {
|
|
135
|
+
return {
|
|
136
|
+
asset_id: record.asset_id,
|
|
137
|
+
stored: false,
|
|
138
|
+
verified,
|
|
139
|
+
status: 'logical_collision',
|
|
140
|
+
...(resultLogicalId(logicalId) ? { logicalId } : {}),
|
|
141
|
+
collisionWithAssetId: collision.asset_id,
|
|
142
|
+
};
|
|
143
|
+
}
|
|
109
144
|
appendUtf8Durable(file, `${JSON.stringify(record)}\n`);
|
|
110
145
|
this.index.set(record.asset_id, record);
|
|
111
146
|
this.updateFileStateAfterWrite();
|
|
@@ -113,22 +148,58 @@ export class LocalJsonlProvider {
|
|
|
113
148
|
finally {
|
|
114
149
|
releaseLock(this.lockPath);
|
|
115
150
|
}
|
|
116
|
-
return {
|
|
151
|
+
return {
|
|
152
|
+
asset_id: record.asset_id,
|
|
153
|
+
stored: true,
|
|
154
|
+
verified,
|
|
155
|
+
status: 'stored',
|
|
156
|
+
...(collision ? {
|
|
157
|
+
...(resultLogicalId(logicalId) ? { logicalId } : {}),
|
|
158
|
+
collisionWithAssetId: collision.asset_id,
|
|
159
|
+
} : {}),
|
|
160
|
+
};
|
|
117
161
|
}
|
|
118
162
|
/**
|
|
119
163
|
* 迁移专用(M8-2): 以**冻结 asset_id** 原样写入, 不经 normalizeForPut 重算/校验.
|
|
120
164
|
* 仅 v1→v2 导入用(硬化 A6 存量冻结); 普通写一律走 put(). record 必须自带 asset_id.
|
|
121
165
|
*/
|
|
122
166
|
async putFrozen(record) {
|
|
167
|
+
return this.putFrozenConditional(record, { allowLogicalCollision: true });
|
|
168
|
+
}
|
|
169
|
+
async putFrozenConditional(record, options = {}) {
|
|
123
170
|
if (!record.asset_id)
|
|
124
171
|
throw new Error('putFrozen 需 record 自带冻结 asset_id');
|
|
125
172
|
const file = join(this.baseDir, LOCAL_ASSET_FILES[record.type]);
|
|
173
|
+
let logicalId;
|
|
174
|
+
let collision;
|
|
126
175
|
assertOptionalRegularFile(this.lockPath, 'lock_file');
|
|
127
176
|
acquireLock(this.lockPath);
|
|
128
177
|
try {
|
|
129
178
|
this.refreshUnderLock();
|
|
130
|
-
|
|
131
|
-
|
|
179
|
+
const existing = this.index.get(record.asset_id);
|
|
180
|
+
if (existing) {
|
|
181
|
+
if (!frozenAssetRecordsEqual(existing, record))
|
|
182
|
+
throw new FrozenAssetIdCollisionError(record.asset_id);
|
|
183
|
+
return { asset_id: record.asset_id, stored: false, verified: false, status: 'already_exists' };
|
|
184
|
+
}
|
|
185
|
+
logicalId = typeof record['id'] === 'string' && record['id'].length > 0
|
|
186
|
+
? record['id']
|
|
187
|
+
: undefined;
|
|
188
|
+
collision = logicalId === undefined
|
|
189
|
+
? undefined
|
|
190
|
+
: [...this.index.values()].find((candidate) => (candidate.type === record.type
|
|
191
|
+
&& candidate['id'] === logicalId
|
|
192
|
+
&& candidate.asset_id !== record.asset_id));
|
|
193
|
+
if (collision && !options.allowLogicalCollision) {
|
|
194
|
+
return {
|
|
195
|
+
asset_id: record.asset_id,
|
|
196
|
+
stored: false,
|
|
197
|
+
verified: false,
|
|
198
|
+
status: 'logical_collision',
|
|
199
|
+
...(resultLogicalId(logicalId) ? { logicalId } : {}),
|
|
200
|
+
collisionWithAssetId: collision.asset_id,
|
|
201
|
+
};
|
|
202
|
+
}
|
|
132
203
|
appendUtf8Durable(file, `${JSON.stringify(record)}\n`);
|
|
133
204
|
this.index.set(record.asset_id, record);
|
|
134
205
|
this.updateFileStateAfterWrite();
|
|
@@ -136,18 +207,27 @@ export class LocalJsonlProvider {
|
|
|
136
207
|
finally {
|
|
137
208
|
releaseLock(this.lockPath);
|
|
138
209
|
}
|
|
139
|
-
return {
|
|
210
|
+
return {
|
|
211
|
+
asset_id: record.asset_id,
|
|
212
|
+
stored: true,
|
|
213
|
+
verified: false,
|
|
214
|
+
status: 'stored',
|
|
215
|
+
...(collision ? {
|
|
216
|
+
...(resultLogicalId(logicalId) ? { logicalId } : {}),
|
|
217
|
+
collisionWithAssetId: collision.asset_id,
|
|
218
|
+
} : {}),
|
|
219
|
+
};
|
|
140
220
|
}
|
|
141
221
|
async get(assetId) {
|
|
142
222
|
this.ensureFresh();
|
|
143
223
|
return this.index.get(assetId) ?? null;
|
|
144
224
|
}
|
|
145
|
-
async findByLogicalId(id, limit = 2) {
|
|
225
|
+
async findByLogicalId(id, limit = 2, kind) {
|
|
146
226
|
this.ensureFresh();
|
|
147
227
|
const boundedLimit = Number.isFinite(limit) ? Math.max(1, Math.min(1_000, Math.floor(limit))) : 2;
|
|
148
228
|
const out = [];
|
|
149
229
|
for (const record of this.index.values()) {
|
|
150
|
-
if (record['id'] !== id)
|
|
230
|
+
if (record['id'] !== id || (kind !== undefined && record.type !== kind))
|
|
151
231
|
continue;
|
|
152
232
|
out.push(record);
|
|
153
233
|
if (out.length >= boundedLimit)
|
|
@@ -180,8 +260,8 @@ export class LocalJsonlProvider {
|
|
|
180
260
|
if (q.gene && r.gene !== q.gene)
|
|
181
261
|
continue;
|
|
182
262
|
if (q.signalsAny && q.signalsAny.length > 0) {
|
|
183
|
-
const sig = new Set(signalsOf(r));
|
|
184
|
-
if (!q.signalsAny.some((
|
|
263
|
+
const sig = new Set(signalsOf(r).map((signal) => signal.trim().toLowerCase()));
|
|
264
|
+
if (!q.signalsAny.some((signal) => sig.has(signal.trim().toLowerCase())))
|
|
185
265
|
continue;
|
|
186
266
|
}
|
|
187
267
|
if (q.text) {
|