@evomap/evolver-core 2.0.0-beta.2 → 2.0.0-beta.22
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/assets/gep/genes.jsonl +5 -5
- package/dist/algo/bans.d.ts +10 -1
- package/dist/algo/bans.js +56 -6
- package/dist/algo/candidateAssembly.d.ts +25 -2
- package/dist/algo/candidateAssembly.js +171 -26
- package/dist/algo/capabilityCandidates.js +10 -0
- package/dist/algo/conversationSniffer.js +25 -8
- package/dist/algo/cycleEngine.d.ts +63 -5
- package/dist/algo/cycleEngine.js +320 -42
- package/dist/algo/cycleFailureClassifier.d.ts +1 -1
- package/dist/algo/evolutionEvent.d.ts +26 -0
- package/dist/algo/evolutionEvent.js +32 -0
- package/dist/algo/exploration.d.ts +7 -0
- package/dist/algo/exploration.js +16 -3
- package/dist/algo/geneHealth.d.ts +36 -3
- package/dist/algo/geneHealth.js +47 -4
- package/dist/algo/geneIntake.d.ts +46 -1
- package/dist/algo/geneIntake.js +121 -10
- package/dist/algo/geneSelection.d.ts +126 -6
- package/dist/algo/geneSelection.js +479 -36
- package/dist/algo/index.d.ts +6 -1
- package/dist/algo/index.js +6 -1
- package/dist/algo/kautoProjection.d.ts +41 -0
- package/dist/algo/kautoProjection.js +95 -0
- package/dist/algo/kautoValidator.d.ts +68 -0
- package/dist/algo/kautoValidator.js +256 -0
- package/dist/algo/memoryGraph.d.ts +62 -0
- package/dist/algo/memoryGraph.js +86 -0
- package/dist/algo/orchestrator.d.ts +17 -1
- package/dist/algo/orchestrator.js +30 -4
- package/dist/algo/publishEligibility.d.ts +34 -0
- package/dist/algo/publishEligibility.js +52 -0
- package/dist/algo/solidify.d.ts +11 -2
- package/dist/algo/solidify.js +37 -7
- package/dist/algo/ucb1.d.ts +53 -0
- package/dist/algo/ucb1.js +156 -0
- package/dist/assetrepair/hubRejection.d.ts +12 -0
- package/dist/assetrepair/hubRejection.js +109 -0
- package/dist/assetrepair/index.d.ts +2 -0
- package/dist/assetrepair/index.js +2 -0
- package/dist/assetrepair/repair.d.ts +33 -0
- package/dist/assetrepair/repair.js +155 -0
- package/dist/assetstore/assetSidecarRecords.d.ts +28 -0
- package/dist/assetstore/assetSidecarRecords.js +384 -0
- package/dist/assetstore/assetSidecarRecovery.d.ts +48 -0
- package/dist/assetstore/assetSidecarRecovery.js +288 -0
- package/dist/assetstore/assetStoreHealth.d.ts +81 -0
- package/dist/assetstore/assetStoreHealth.js +319 -0
- package/dist/assetstore/assetStoreLayout.d.ts +2 -0
- package/dist/assetstore/assetStoreLayout.js +6 -0
- package/dist/assetstore/assetStoreStorage.d.ts +42 -0
- package/dist/assetstore/assetStoreStorage.js +336 -0
- package/dist/assetstore/assetSyncLedger.d.ts +91 -1
- package/dist/assetstore/assetSyncLedger.js +718 -59
- package/dist/assetstore/foreignJsonlSource.d.ts +48 -0
- package/dist/assetstore/foreignJsonlSource.js +150 -0
- package/dist/assetstore/index.d.ts +5 -0
- package/dist/assetstore/index.js +5 -0
- package/dist/assetstore/learningHistory.js +3 -3
- package/dist/assetstore/localAssetStoreSnapshot.d.ts +51 -0
- package/dist/assetstore/localAssetStoreSnapshot.js +329 -0
- package/dist/assetstore/localJsonl.d.ts +12 -1
- package/dist/assetstore/localJsonl.js +251 -42
- package/dist/assetstore/provenance.d.ts +93 -4
- package/dist/assetstore/provenance.js +368 -84
- package/dist/assetstore/provider.d.ts +63 -0
- package/dist/assetstore/provider.js +97 -6
- package/dist/assetstore/reviewFilter.d.ts +19 -1
- package/dist/assetstore/reviewFilter.js +39 -1
- package/dist/assetstore/reviewLedger.d.ts +8 -2
- package/dist/assetstore/reviewLedger.js +71 -45
- package/dist/assetstore/unionReadStore.d.ts +25 -0
- package/dist/assetstore/unionReadStore.js +119 -0
- package/dist/benchmark/antiGeneBenchmark.d.ts +2 -0
- package/dist/benchmark/antiGeneBenchmark.js +4 -3
- package/dist/benchmark/antiGeneRollout.d.ts +2 -0
- package/dist/benchmark/antiGeneRollout.js +4 -3
- package/dist/benchmark/index.d.ts +3 -1
- package/dist/benchmark/index.js +3 -1
- package/dist/benchmark/selectionFlatAbstention.d.ts +152 -0
- package/dist/benchmark/selectionFlatAbstention.js +481 -0
- package/dist/benchmark/triggerShift.d.ts +62 -0
- package/dist/benchmark/triggerShift.js +106 -0
- package/dist/bootstrap/envFingerprint.d.ts +9 -0
- package/dist/bootstrap/envFingerprint.js +5 -0
- package/dist/bootstrap/index.d.ts +3 -1
- package/dist/bootstrap/index.js +3 -1
- package/dist/bootstrap/lifecycleBootstrap.d.ts +111 -0
- package/dist/bootstrap/lifecycleBootstrap.js +433 -0
- package/dist/bootstrap/v1EnvCompat.d.ts +113 -0
- package/dist/bootstrap/v1EnvCompat.js +303 -0
- package/dist/events/eventArchive.d.ts +2 -0
- package/dist/events/eventArchive.js +13 -3
- package/dist/events/eventSchema.d.ts +7 -7
- package/dist/events/eventStore.d.ts +2 -0
- package/dist/events/eventStore.js +5 -1
- package/dist/events/ingest.d.ts +2 -1
- package/dist/events/ingest.js +15 -0
- package/dist/events/paths.d.ts +12 -10
- package/dist/events/paths.js +24 -20
- package/dist/events/public.d.ts +2 -2
- package/dist/events/public.js +2 -2
- package/dist/events/reports.d.ts +2 -0
- package/dist/events/reports.js +4 -0
- package/dist/exec/autoExec.d.ts +68 -4
- package/dist/exec/autoExec.js +392 -33
- package/dist/exec/autonomousCycle.d.ts +31 -4
- package/dist/exec/autonomousCycle.js +71 -13
- package/dist/exec/claudeBridge.d.ts +80 -15
- package/dist/exec/claudeBridge.js +845 -76
- package/dist/exec/executionBinding.d.ts +414 -0
- package/dist/exec/executionBinding.js +588 -0
- package/dist/exec/index.d.ts +1 -0
- package/dist/exec/index.js +1 -0
- package/dist/exec/openPrRegistry.d.ts +8 -2
- package/dist/exec/openPrRegistry.js +32 -22
- package/dist/exec/prompt.js +14 -1
- package/dist/exec/proofOfWork.d.ts +1 -1
- package/dist/exec/proofOfWork.js +2 -2
- package/dist/exec/runnerRegistry.d.ts +153 -36
- package/dist/exec/runnerRegistry.js +848 -65
- package/dist/exec/selfPr.js +1 -7
- package/dist/feedback/envelope.d.ts +61 -0
- package/dist/feedback/envelope.js +168 -0
- package/dist/feedback/index.d.ts +1 -0
- package/dist/feedback/index.js +1 -0
- package/dist/hooks/hooks.js +1 -0
- package/dist/hub/assetCallLog.d.ts +35 -1
- package/dist/hub/assetCallLog.js +124 -1
- package/dist/hub/bindings.d.ts +8 -1
- package/dist/hub/bindings.js +29 -8
- package/dist/hub/capability.d.ts +130 -4
- package/dist/hub/conversationDistiller.d.ts +19 -0
- package/dist/hub/conversationDistiller.js +115 -37
- package/dist/hub/fake.d.ts +3 -2
- package/dist/hub/fake.js +2 -1
- package/dist/hub/index.d.ts +1 -0
- package/dist/hub/index.js +1 -0
- package/dist/hub/questionGenerator.d.ts +5 -1
- package/dist/hub/questionGenerator.js +8 -6
- package/dist/hub/recipeCompose.d.ts +27 -0
- package/dist/hub/recipeCompose.js +90 -0
- package/dist/hub/sanitize.js +122 -7
- package/dist/index.d.ts +6 -1
- package/dist/index.js +7 -1
- package/dist/issueReporter/index.d.ts +156 -0
- package/dist/issueReporter/index.js +1688 -0
- package/dist/mailbox/dispatch.d.ts +1 -1
- package/dist/mailbox/dispatch.js +22 -6
- package/dist/mailbox/envelope.d.ts +7 -1
- package/dist/mailbox/envelope.js +9 -2
- package/dist/mailbox/ipcServer.d.ts +12 -2
- package/dist/mailbox/ipcServer.js +183 -13
- package/dist/mailbox/store.d.ts +89 -3
- package/dist/mailbox/store.js +895 -41
- package/dist/modelCompatibility.d.ts +164 -0
- package/dist/modelCompatibility.js +309 -0
- package/dist/observers/valueDigestObserver.d.ts +9 -0
- package/dist/observers/valueDigestObserver.js +35 -2
- package/dist/ops/cleanup.js +1 -1
- package/dist/ops/evolutionGraphProjection.d.ts +20 -0
- package/dist/ops/evolutionGraphProjection.js +315 -0
- package/dist/ops/index.d.ts +2 -1
- package/dist/ops/index.js +2 -1
- package/dist/ops/savingsCore.js +1 -2
- package/dist/ops/selfUpdate.d.ts +18 -1
- package/dist/ops/selfUpdate.js +88 -23
- package/dist/ops/valueOutreach.d.ts +3 -1
- package/dist/ops/valueOutreach.js +5 -1
- package/dist/personality/schema.d.ts +24 -24
- package/dist/schema/evolutionGraph.d.ts +784 -0
- package/dist/schema/evolutionGraph.js +187 -0
- package/dist/schema/index.d.ts +1 -0
- package/dist/schema/index.js +1 -0
- package/dist/schema/proofOfWork.d.ts +125 -6
- package/dist/schema/proofOfWork.js +102 -4
- package/dist/schema/signal.d.ts +3 -3
- package/dist/schema/signal.js +1 -1
- package/dist/shadow/shadowHub.js +1 -0
- package/dist/signals/curriculum.d.ts +55 -0
- package/dist/signals/curriculum.js +202 -0
- package/dist/signals/cycleHistoryFromEvents.js +17 -8
- package/dist/signals/expand.d.ts +15 -1
- package/dist/signals/expand.js +169 -1
- package/dist/signals/extractor.d.ts +2 -2
- package/dist/signals/extractor.js +31 -6
- package/dist/signals/index.d.ts +4 -1
- package/dist/signals/index.js +4 -1
- package/dist/signals/metaSignals.d.ts +4 -0
- package/dist/signals/metaSignals.js +42 -0
- package/dist/signals/scopeVocabulary.d.ts +75 -0
- package/dist/signals/scopeVocabulary.js +91 -0
- package/dist/signals/signalGate.js +1 -1
- package/dist/signals/taskDomain.d.ts +22 -0
- package/dist/signals/taskDomain.js +43 -0
- package/dist/strategy/constraintAblation.d.ts +64 -0
- package/dist/strategy/constraintAblation.js +2820 -0
- package/dist/strategy/constraintAblationPredicates.d.ts +31 -0
- package/dist/strategy/constraintAblationPredicates.js +339 -0
- package/dist/strategy/index.d.ts +2 -1
- package/dist/strategy/index.js +2 -1
- package/dist/trace/index.d.ts +3 -1
- package/dist/trace/index.js +3 -1
- package/dist/trace/learningTrace.d.ts +216 -0
- package/dist/trace/learningTrace.js +298 -0
- package/dist/trace/proxyTurns.d.ts +31 -0
- package/dist/trace/proxyTurns.js +137 -0
- package/dist/trace/trajectory.d.ts +8 -0
- package/dist/trace/trajectory.js +14 -2
- package/dist/util/fetchPort.d.ts +1 -0
- package/dist/util/fetchPort.js +11 -0
- package/dist/util/fileLock.d.ts +113 -7
- package/dist/util/fileLock.js +1035 -80
- package/dist/util/index.d.ts +3 -1
- package/dist/util/index.js +2 -1
- package/dist/verify/index.d.ts +2 -1
- package/dist/verify/index.js +1 -1
- package/dist/verify/sandboxRunner.d.ts +30 -0
- package/dist/verify/sandboxRunner.js +370 -27
- package/dist/verify/sandboxedValidation.d.ts +16 -2
- package/dist/verify/sandboxedValidation.js +200 -22
- package/dist/verify/validation.d.ts +35 -4
- package/dist/verify/validation.js +164 -16
- package/dist/wire/geneHints.d.ts +88 -16
- package/dist/wire/geneHints.js +124 -15
- package/dist/wire/index.d.ts +8 -3
- package/dist/wire/index.js +2 -2
- package/dist/wire/schemaGate.d.ts +21 -0
- package/dist/wire/schemaGate.js +119 -10
- package/dist/workflow/dsl.d.ts +24 -3
- package/dist/workflow/dsl.js +4 -0
- package/dist/workflow/engine.d.ts +5 -1
- package/dist/workflow/engine.js +3 -0
- package/dist/workflow/index.d.ts +3 -1
- package/dist/workflow/index.js +3 -1
- package/dist/workflow/runtime.d.ts +110 -0
- package/dist/workflow/runtime.js +1298 -0
- package/dist/workflow/stateStore.d.ts +172 -0
- package/dist/workflow/stateStore.js +1044 -0
- package/package.json +12 -4
|
@@ -1,23 +1,20 @@
|
|
|
1
|
-
import {
|
|
1
|
+
import { unlinkSync } from 'node:fs';
|
|
2
2
|
import { join } from 'node:path';
|
|
3
3
|
import { acquireLock, releaseLock } from '../util/fileLock.js';
|
|
4
|
-
import {
|
|
5
|
-
|
|
6
|
-
|
|
7
|
-
|
|
4
|
+
import { appendUtf8Durable, assertAssetStoreDirectory, assertOptionalRegularFile, ensureAssetStoreDirectory, fsyncDirectoryBestEffort, readUtf8Regular, regularFileFingerprint, replaceUtf8Durable, } from './assetStoreStorage.js';
|
|
5
|
+
import { LOCAL_ASSET_FILES } from './assetStoreLayout.js';
|
|
6
|
+
import { FrozenAssetIdCollisionError, frozenAssetRecordsEqual, normalizeForPut, } from './provider.js';
|
|
7
|
+
function resultLogicalId(logicalId) {
|
|
8
|
+
return logicalId !== undefined && logicalId.length > 0 && logicalId === logicalId.trim()
|
|
9
|
+
? logicalId
|
|
10
|
+
: undefined;
|
|
8
11
|
}
|
|
9
|
-
|
|
10
|
-
|
|
11
|
-
|
|
12
|
-
|
|
13
|
-
|
|
14
|
-
|
|
15
|
-
if (isErrno(error, 'ENOENT'))
|
|
16
|
-
return 'missing';
|
|
17
|
-
throw error;
|
|
18
|
-
}
|
|
19
|
-
}
|
|
20
|
-
function signalsOf(a) {
|
|
12
|
+
const BUNDLE_JOURNAL_FILE = '.assetstore-bundle.json';
|
|
13
|
+
const BUNDLE_JOURNAL_SCHEMA = 'evolver.assetstore-bundle.v1';
|
|
14
|
+
const MAX_BUNDLE_ASSETS = 64;
|
|
15
|
+
const MAX_BUNDLE_JOURNAL_BYTES = 4 * 1024 * 1024;
|
|
16
|
+
/** Signal names a record advertises, across the four key spellings the pool uses. */
|
|
17
|
+
export function signalsOf(a) {
|
|
21
18
|
const out = [];
|
|
22
19
|
for (const key of ['signals_match', 'signals', 'trigger', 'trigger_signals']) {
|
|
23
20
|
const v = a[key];
|
|
@@ -38,19 +35,22 @@ export class LocalJsonlProvider {
|
|
|
38
35
|
baseDir;
|
|
39
36
|
index = new Map();
|
|
40
37
|
lockPath;
|
|
38
|
+
bundleJournalPath;
|
|
41
39
|
fileState = new Map();
|
|
42
40
|
loaded = false;
|
|
43
41
|
// `baseDir` is public-readonly so callers that inject a store (e.g. the CLI under test) can co-locate sidecars
|
|
44
42
|
// — the ReviewLedger/ProvenanceStore — in the SAME directory, instead of defaulting to the real ~/.evomap.
|
|
45
43
|
constructor(baseDir) {
|
|
46
44
|
this.baseDir = baseDir;
|
|
47
|
-
|
|
45
|
+
ensureAssetStoreDirectory(baseDir);
|
|
48
46
|
this.lockPath = join(baseDir, '.assetstore.lock');
|
|
47
|
+
this.bundleJournalPath = join(baseDir, BUNDLE_JOURNAL_FILE);
|
|
49
48
|
}
|
|
50
49
|
captureFileState() {
|
|
50
|
+
assertAssetStoreDirectory(this.baseDir);
|
|
51
51
|
const state = new Map();
|
|
52
|
-
for (const [kind, file] of Object.entries(
|
|
53
|
-
state.set(kind,
|
|
52
|
+
for (const [kind, file] of Object.entries(LOCAL_ASSET_FILES)) {
|
|
53
|
+
state.set(kind, regularFileFingerprint(join(this.baseDir, file)));
|
|
54
54
|
}
|
|
55
55
|
return state;
|
|
56
56
|
}
|
|
@@ -65,11 +65,12 @@ export class LocalJsonlProvider {
|
|
|
65
65
|
}
|
|
66
66
|
rebuildIndex(state) {
|
|
67
67
|
const next = new Map();
|
|
68
|
-
for (const file of Object.values(
|
|
68
|
+
for (const file of Object.values(LOCAL_ASSET_FILES)) {
|
|
69
69
|
const p = join(this.baseDir, file);
|
|
70
|
-
|
|
70
|
+
const raw = readUtf8Regular(p);
|
|
71
|
+
if (raw === null)
|
|
71
72
|
continue;
|
|
72
|
-
for (const line of
|
|
73
|
+
for (const line of raw.split('\n')) {
|
|
73
74
|
if (!line.trim())
|
|
74
75
|
continue;
|
|
75
76
|
try {
|
|
@@ -93,10 +94,12 @@ export class LocalJsonlProvider {
|
|
|
93
94
|
}
|
|
94
95
|
ensureFresh() {
|
|
95
96
|
const state = this.captureFileState();
|
|
96
|
-
if (!this.stateChanged(state))
|
|
97
|
+
if (!this.hasPendingBundle() && !this.stateChanged(state))
|
|
97
98
|
return;
|
|
99
|
+
assertOptionalRegularFile(this.lockPath, 'lock_file');
|
|
98
100
|
acquireLock(this.lockPath);
|
|
99
101
|
try {
|
|
102
|
+
this.recoverPendingBundleUnderLock();
|
|
100
103
|
this.refreshUnderLock();
|
|
101
104
|
}
|
|
102
105
|
finally {
|
|
@@ -107,50 +110,252 @@ export class LocalJsonlProvider {
|
|
|
107
110
|
this.fileState = this.captureFileState();
|
|
108
111
|
this.loaded = true;
|
|
109
112
|
}
|
|
113
|
+
readPendingBundle() {
|
|
114
|
+
if (assertOptionalRegularFile(this.bundleJournalPath, 'temp_file') === null)
|
|
115
|
+
return undefined;
|
|
116
|
+
const raw = readUtf8Regular(this.bundleJournalPath, MAX_BUNDLE_JOURNAL_BYTES);
|
|
117
|
+
if (raw === null)
|
|
118
|
+
return undefined;
|
|
119
|
+
let parsed;
|
|
120
|
+
try {
|
|
121
|
+
parsed = JSON.parse(raw);
|
|
122
|
+
}
|
|
123
|
+
catch {
|
|
124
|
+
throw new Error('asset store bundle journal is malformed');
|
|
125
|
+
}
|
|
126
|
+
if (!parsed || typeof parsed !== 'object' || Array.isArray(parsed)) {
|
|
127
|
+
throw new Error('asset store bundle journal is malformed');
|
|
128
|
+
}
|
|
129
|
+
const journal = parsed;
|
|
130
|
+
const assets = journal['assets'];
|
|
131
|
+
if (journal['schema'] !== BUNDLE_JOURNAL_SCHEMA || !Array.isArray(assets) || assets.length === 0 || assets.length > MAX_BUNDLE_ASSETS) {
|
|
132
|
+
throw new Error('asset store bundle journal is invalid');
|
|
133
|
+
}
|
|
134
|
+
return assets.map((asset) => {
|
|
135
|
+
if (!asset || typeof asset !== 'object' || Array.isArray(asset))
|
|
136
|
+
throw new Error('asset store bundle journal is invalid');
|
|
137
|
+
const record = asset;
|
|
138
|
+
if (typeof record.type !== 'string' || !Object.prototype.hasOwnProperty.call(LOCAL_ASSET_FILES, record.type)
|
|
139
|
+
|| typeof record.asset_id !== 'string' || record.asset_id.length === 0) {
|
|
140
|
+
throw new Error('asset store bundle journal is invalid');
|
|
141
|
+
}
|
|
142
|
+
return record;
|
|
143
|
+
});
|
|
144
|
+
}
|
|
145
|
+
recoverPendingBundleUnderLock() {
|
|
146
|
+
const pending = this.readPendingBundle();
|
|
147
|
+
if (!pending)
|
|
148
|
+
return;
|
|
149
|
+
this.refreshUnderLock();
|
|
150
|
+
for (const pendingRecord of pending) {
|
|
151
|
+
const { record } = normalizeForPut(pendingRecord);
|
|
152
|
+
if (record.asset_id !== pendingRecord.asset_id)
|
|
153
|
+
throw new Error('asset store bundle journal asset_id mismatch');
|
|
154
|
+
const existing = this.index.get(record.asset_id);
|
|
155
|
+
if (existing) {
|
|
156
|
+
if (!frozenAssetRecordsEqual(existing, record))
|
|
157
|
+
throw new FrozenAssetIdCollisionError(record.asset_id);
|
|
158
|
+
continue;
|
|
159
|
+
}
|
|
160
|
+
appendUtf8Durable(join(this.baseDir, LOCAL_ASSET_FILES[record.type]), `${JSON.stringify(record)}\n`);
|
|
161
|
+
this.index.set(record.asset_id, record);
|
|
162
|
+
}
|
|
163
|
+
unlinkSync(this.bundleJournalPath);
|
|
164
|
+
fsyncDirectoryBestEffort(this.baseDir);
|
|
165
|
+
this.updateFileStateAfterWrite();
|
|
166
|
+
}
|
|
167
|
+
hasPendingBundle() {
|
|
168
|
+
return assertOptionalRegularFile(this.bundleJournalPath, 'temp_file') !== null;
|
|
169
|
+
}
|
|
110
170
|
async put(asset) {
|
|
171
|
+
return this.putConditional(asset, { allowLogicalCollision: true });
|
|
172
|
+
}
|
|
173
|
+
async putConditional(asset, options) {
|
|
111
174
|
const { record, verified } = normalizeForPut(asset);
|
|
112
|
-
const file = join(this.baseDir,
|
|
175
|
+
const file = join(this.baseDir, LOCAL_ASSET_FILES[record.type]);
|
|
176
|
+
const logicalId = typeof record.id === 'string' ? record.id : undefined;
|
|
177
|
+
let collision;
|
|
178
|
+
assertOptionalRegularFile(this.lockPath, 'lock_file');
|
|
113
179
|
acquireLock(this.lockPath);
|
|
114
180
|
try {
|
|
181
|
+
this.recoverPendingBundleUnderLock();
|
|
115
182
|
// Refresh under the shared lock so another process cannot append between reload and dedupe.
|
|
116
183
|
this.refreshUnderLock();
|
|
117
|
-
|
|
118
|
-
|
|
119
|
-
|
|
184
|
+
const existing = this.index.get(record.asset_id);
|
|
185
|
+
if (existing) {
|
|
186
|
+
if (!frozenAssetRecordsEqual(existing, record))
|
|
187
|
+
throw new FrozenAssetIdCollisionError(record.asset_id);
|
|
188
|
+
return {
|
|
189
|
+
asset_id: record.asset_id,
|
|
190
|
+
stored: false,
|
|
191
|
+
verified,
|
|
192
|
+
status: 'already_exists',
|
|
193
|
+
...(resultLogicalId(logicalId) ? { logicalId } : {}),
|
|
194
|
+
};
|
|
195
|
+
}
|
|
196
|
+
collision = logicalId === undefined
|
|
197
|
+
? undefined
|
|
198
|
+
: [...this.index.values()].find((existing) => (existing.type === record.type
|
|
199
|
+
&& existing.id === logicalId
|
|
200
|
+
&& existing.asset_id !== record.asset_id));
|
|
201
|
+
if (collision && !options?.allowLogicalCollision) {
|
|
202
|
+
return {
|
|
203
|
+
asset_id: record.asset_id,
|
|
204
|
+
stored: false,
|
|
205
|
+
verified,
|
|
206
|
+
status: 'logical_collision',
|
|
207
|
+
...(resultLogicalId(logicalId) ? { logicalId } : {}),
|
|
208
|
+
collisionWithAssetId: collision.asset_id,
|
|
209
|
+
};
|
|
210
|
+
}
|
|
211
|
+
appendUtf8Durable(file, `${JSON.stringify(record)}\n`);
|
|
120
212
|
this.index.set(record.asset_id, record);
|
|
121
213
|
this.updateFileStateAfterWrite();
|
|
122
214
|
}
|
|
123
215
|
finally {
|
|
124
216
|
releaseLock(this.lockPath);
|
|
125
217
|
}
|
|
126
|
-
return {
|
|
218
|
+
return {
|
|
219
|
+
asset_id: record.asset_id,
|
|
220
|
+
stored: true,
|
|
221
|
+
verified,
|
|
222
|
+
status: 'stored',
|
|
223
|
+
...(collision ? {
|
|
224
|
+
...(resultLogicalId(logicalId) ? { logicalId } : {}),
|
|
225
|
+
collisionWithAssetId: collision.asset_id,
|
|
226
|
+
} : {}),
|
|
227
|
+
};
|
|
228
|
+
}
|
|
229
|
+
async putBundle(assets) {
|
|
230
|
+
if (assets.length === 0)
|
|
231
|
+
return [];
|
|
232
|
+
if (assets.length > MAX_BUNDLE_ASSETS)
|
|
233
|
+
throw new Error('asset store bundle is too large');
|
|
234
|
+
const normalized = assets.map((asset) => normalizeForPut(asset));
|
|
235
|
+
const seen = new Map();
|
|
236
|
+
for (const normalizedRecord of normalized) {
|
|
237
|
+
const { record } = normalizedRecord;
|
|
238
|
+
const previous = seen.get(record.asset_id);
|
|
239
|
+
if (previous && !frozenAssetRecordsEqual(previous.record, record))
|
|
240
|
+
throw new FrozenAssetIdCollisionError(record.asset_id);
|
|
241
|
+
if (!previous)
|
|
242
|
+
seen.set(record.asset_id, normalizedRecord);
|
|
243
|
+
}
|
|
244
|
+
const results = [];
|
|
245
|
+
assertOptionalRegularFile(this.lockPath, 'lock_file');
|
|
246
|
+
acquireLock(this.lockPath);
|
|
247
|
+
try {
|
|
248
|
+
this.recoverPendingBundleUnderLock();
|
|
249
|
+
this.refreshUnderLock();
|
|
250
|
+
const pending = [];
|
|
251
|
+
for (const { record, verified } of seen.values()) {
|
|
252
|
+
const existing = this.index.get(record.asset_id);
|
|
253
|
+
if (existing) {
|
|
254
|
+
if (!frozenAssetRecordsEqual(existing, record))
|
|
255
|
+
throw new FrozenAssetIdCollisionError(record.asset_id);
|
|
256
|
+
results.push({ asset_id: record.asset_id, stored: false, verified });
|
|
257
|
+
continue;
|
|
258
|
+
}
|
|
259
|
+
pending.push(record);
|
|
260
|
+
results.push({ asset_id: record.asset_id, stored: true, verified });
|
|
261
|
+
}
|
|
262
|
+
if (pending.length === 0)
|
|
263
|
+
return results;
|
|
264
|
+
const journal = `${JSON.stringify({ schema: BUNDLE_JOURNAL_SCHEMA, assets: pending })}\n`;
|
|
265
|
+
if (Buffer.byteLength(journal, 'utf8') > MAX_BUNDLE_JOURNAL_BYTES) {
|
|
266
|
+
throw new Error('asset store bundle is too large');
|
|
267
|
+
}
|
|
268
|
+
replaceUtf8Durable(this.bundleJournalPath, journal);
|
|
269
|
+
for (const record of pending) {
|
|
270
|
+
appendUtf8Durable(join(this.baseDir, LOCAL_ASSET_FILES[record.type]), `${JSON.stringify(record)}\n`);
|
|
271
|
+
this.index.set(record.asset_id, record);
|
|
272
|
+
}
|
|
273
|
+
unlinkSync(this.bundleJournalPath);
|
|
274
|
+
fsyncDirectoryBestEffort(this.baseDir);
|
|
275
|
+
this.updateFileStateAfterWrite();
|
|
276
|
+
}
|
|
277
|
+
finally {
|
|
278
|
+
releaseLock(this.lockPath);
|
|
279
|
+
}
|
|
280
|
+
return results;
|
|
127
281
|
}
|
|
128
282
|
/**
|
|
129
283
|
* 迁移专用(M8-2): 以**冻结 asset_id** 原样写入, 不经 normalizeForPut 重算/校验.
|
|
130
284
|
* 仅 v1→v2 导入用(硬化 A6 存量冻结); 普通写一律走 put(). record 必须自带 asset_id.
|
|
131
285
|
*/
|
|
132
286
|
async putFrozen(record) {
|
|
287
|
+
return this.putFrozenConditional(record, { allowLogicalCollision: true });
|
|
288
|
+
}
|
|
289
|
+
async putFrozenConditional(record, options = {}) {
|
|
133
290
|
if (!record.asset_id)
|
|
134
291
|
throw new Error('putFrozen 需 record 自带冻结 asset_id');
|
|
135
|
-
const file = join(this.baseDir,
|
|
292
|
+
const file = join(this.baseDir, LOCAL_ASSET_FILES[record.type]);
|
|
293
|
+
let logicalId;
|
|
294
|
+
let collision;
|
|
295
|
+
assertOptionalRegularFile(this.lockPath, 'lock_file');
|
|
136
296
|
acquireLock(this.lockPath);
|
|
137
297
|
try {
|
|
298
|
+
this.recoverPendingBundleUnderLock();
|
|
138
299
|
this.refreshUnderLock();
|
|
139
|
-
|
|
140
|
-
|
|
141
|
-
|
|
300
|
+
const existing = this.index.get(record.asset_id);
|
|
301
|
+
if (existing) {
|
|
302
|
+
if (!frozenAssetRecordsEqual(existing, record))
|
|
303
|
+
throw new FrozenAssetIdCollisionError(record.asset_id);
|
|
304
|
+
return { asset_id: record.asset_id, stored: false, verified: false, status: 'already_exists' };
|
|
305
|
+
}
|
|
306
|
+
logicalId = typeof record['id'] === 'string' && record['id'].length > 0
|
|
307
|
+
? record['id']
|
|
308
|
+
: undefined;
|
|
309
|
+
collision = logicalId === undefined
|
|
310
|
+
? undefined
|
|
311
|
+
: [...this.index.values()].find((candidate) => (candidate.type === record.type
|
|
312
|
+
&& candidate['id'] === logicalId
|
|
313
|
+
&& candidate.asset_id !== record.asset_id));
|
|
314
|
+
if (collision && !options.allowLogicalCollision) {
|
|
315
|
+
return {
|
|
316
|
+
asset_id: record.asset_id,
|
|
317
|
+
stored: false,
|
|
318
|
+
verified: false,
|
|
319
|
+
status: 'logical_collision',
|
|
320
|
+
...(resultLogicalId(logicalId) ? { logicalId } : {}),
|
|
321
|
+
collisionWithAssetId: collision.asset_id,
|
|
322
|
+
};
|
|
323
|
+
}
|
|
324
|
+
appendUtf8Durable(file, `${JSON.stringify(record)}\n`);
|
|
142
325
|
this.index.set(record.asset_id, record);
|
|
143
326
|
this.updateFileStateAfterWrite();
|
|
144
327
|
}
|
|
145
328
|
finally {
|
|
146
329
|
releaseLock(this.lockPath);
|
|
147
330
|
}
|
|
148
|
-
return {
|
|
331
|
+
return {
|
|
332
|
+
asset_id: record.asset_id,
|
|
333
|
+
stored: true,
|
|
334
|
+
verified: false,
|
|
335
|
+
status: 'stored',
|
|
336
|
+
...(collision ? {
|
|
337
|
+
...(resultLogicalId(logicalId) ? { logicalId } : {}),
|
|
338
|
+
collisionWithAssetId: collision.asset_id,
|
|
339
|
+
} : {}),
|
|
340
|
+
};
|
|
149
341
|
}
|
|
150
342
|
async get(assetId) {
|
|
151
343
|
this.ensureFresh();
|
|
152
344
|
return this.index.get(assetId) ?? null;
|
|
153
345
|
}
|
|
346
|
+
async findByLogicalId(id, limit = 2, kind) {
|
|
347
|
+
this.ensureFresh();
|
|
348
|
+
const boundedLimit = Number.isFinite(limit) ? Math.max(1, Math.min(1_000, Math.floor(limit))) : 2;
|
|
349
|
+
const out = [];
|
|
350
|
+
for (const record of this.index.values()) {
|
|
351
|
+
if (record['id'] !== id || (kind !== undefined && record.type !== kind))
|
|
352
|
+
continue;
|
|
353
|
+
out.push(record);
|
|
354
|
+
if (out.length >= boundedLimit)
|
|
355
|
+
break;
|
|
356
|
+
}
|
|
357
|
+
return out;
|
|
358
|
+
}
|
|
154
359
|
async list(kind, limit = 1000) {
|
|
155
360
|
this.ensureFresh();
|
|
156
361
|
const out = [];
|
|
@@ -162,6 +367,10 @@ export class LocalJsonlProvider {
|
|
|
162
367
|
}
|
|
163
368
|
return out;
|
|
164
369
|
}
|
|
370
|
+
listAll(kind) {
|
|
371
|
+
this.ensureFresh();
|
|
372
|
+
return [...this.index.values()].filter((record) => kind === undefined || record.type === kind);
|
|
373
|
+
}
|
|
165
374
|
async search(q) {
|
|
166
375
|
this.ensureFresh();
|
|
167
376
|
const out = [];
|
|
@@ -176,8 +385,8 @@ export class LocalJsonlProvider {
|
|
|
176
385
|
if (q.gene && r.gene !== q.gene)
|
|
177
386
|
continue;
|
|
178
387
|
if (q.signalsAny && q.signalsAny.length > 0) {
|
|
179
|
-
const sig = new Set(signalsOf(r));
|
|
180
|
-
if (!q.signalsAny.some((
|
|
388
|
+
const sig = new Set(signalsOf(r).map((signal) => signal.trim().toLowerCase()));
|
|
389
|
+
if (!q.signalsAny.some((signal) => sig.has(signal.trim().toLowerCase())))
|
|
181
390
|
continue;
|
|
182
391
|
}
|
|
183
392
|
if (q.text) {
|
|
@@ -199,15 +408,17 @@ export class LocalJsonlProvider {
|
|
|
199
408
|
* the write lock. Returns kept/removed line counts.
|
|
200
409
|
*/
|
|
201
410
|
async compact() {
|
|
411
|
+
assertOptionalRegularFile(this.lockPath, 'lock_file');
|
|
202
412
|
acquireLock(this.lockPath);
|
|
203
413
|
try {
|
|
204
414
|
this.refreshUnderLock();
|
|
205
415
|
let kept = 0, removed = 0;
|
|
206
|
-
for (const [kind, file] of Object.entries(
|
|
416
|
+
for (const [kind, file] of Object.entries(LOCAL_ASSET_FILES)) {
|
|
207
417
|
const p = join(this.baseDir, file);
|
|
208
|
-
|
|
418
|
+
const raw = readUtf8Regular(p);
|
|
419
|
+
if (raw === null)
|
|
209
420
|
continue;
|
|
210
|
-
const lines =
|
|
421
|
+
const lines = raw.split('\n').filter((l) => l.trim());
|
|
211
422
|
const byId = new Map(); // asset_id → raw line (last wins, matching load semantics)
|
|
212
423
|
for (const line of lines) {
|
|
213
424
|
try {
|
|
@@ -218,9 +429,7 @@ export class LocalJsonlProvider {
|
|
|
218
429
|
catch { /* drop corrupt line */ }
|
|
219
430
|
}
|
|
220
431
|
const out = byId.size ? `${[...byId.values()].join('\n')}\n` : '';
|
|
221
|
-
|
|
222
|
-
writeFileSync(tmp, out);
|
|
223
|
-
renameSync(tmp, p); // atomic replace; a crash mid-compact leaves the original intact
|
|
432
|
+
replaceUtf8Durable(p, out);
|
|
224
433
|
kept += byId.size;
|
|
225
434
|
removed += lines.length - byId.size;
|
|
226
435
|
}
|
|
@@ -1,17 +1,67 @@
|
|
|
1
|
-
import { type AssetStoreProvider, type AssetRecord, type PutResult } from './provider.js';
|
|
1
|
+
import { type AssetStoreProvider, type AssetRecord, type ConditionalPutOptions, type ConditionalPutResult, type PutResult } from './provider.js';
|
|
2
2
|
export type ProvenanceSource = 'local' | 'migrated' | 'hub';
|
|
3
|
+
export type ProvenanceDecision = 'promoted' | 'revoked';
|
|
4
|
+
export declare const UNVERIFIED_V1_IMPORT_REASON = "unverified_v1_import";
|
|
3
5
|
export interface ProvenanceRecord {
|
|
4
6
|
assetId: string;
|
|
5
7
|
source: ProvenanceSource;
|
|
6
8
|
trusted: boolean;
|
|
7
9
|
at: string;
|
|
10
|
+
decision?: ProvenanceDecision;
|
|
11
|
+
decidedBy?: string;
|
|
12
|
+
/** Legacy promotion actor field kept for existing sidecar readers. */
|
|
8
13
|
promotedBy?: string;
|
|
9
14
|
reason?: string;
|
|
15
|
+
/** Canonical content id of the exact hash-mismatched body accepted by a supported unverified ingest. */
|
|
16
|
+
frozenContentId?: string;
|
|
17
|
+
}
|
|
18
|
+
export interface ProvenanceTrustChange {
|
|
19
|
+
changed: boolean;
|
|
20
|
+
record: ProvenanceRecord;
|
|
21
|
+
}
|
|
22
|
+
export type UnverifiedIngestDisposition = {
|
|
23
|
+
readonly status: 'create';
|
|
24
|
+
readonly assetId: string;
|
|
25
|
+
readonly frozenContentId: string;
|
|
26
|
+
readonly provenanceAction: 'stage_and_finalize';
|
|
27
|
+
} | {
|
|
28
|
+
readonly status: 'already_bound';
|
|
29
|
+
readonly assetId: string;
|
|
30
|
+
readonly frozenContentId: string;
|
|
31
|
+
readonly provenanceAction: 'finalize' | 'none';
|
|
32
|
+
} | {
|
|
33
|
+
readonly status: 'preserve_trust';
|
|
34
|
+
readonly assetId: string;
|
|
35
|
+
readonly frozenContentId: string;
|
|
36
|
+
readonly provenanceAction: 'none';
|
|
37
|
+
} | {
|
|
38
|
+
readonly status: 'collision';
|
|
39
|
+
readonly assetId: string;
|
|
40
|
+
readonly frozenContentId: string;
|
|
41
|
+
readonly collision: 'asset' | 'provenance';
|
|
42
|
+
readonly provenanceAction: 'none';
|
|
43
|
+
};
|
|
44
|
+
/**
|
|
45
|
+
* Resolve the exact no-I/O disposition used by {@link ingestUnverified}. Callers may inspect a read-only
|
|
46
|
+
* target snapshot during migration planning, then apply through ingestUnverified without duplicating its
|
|
47
|
+
* trust and collision rules.
|
|
48
|
+
*/
|
|
49
|
+
export declare function planUnverifiedIngest(record: AssetRecord, existing: AssetRecord | null, currentProvenance: ProvenanceRecord | null, reason: string, source?: ProvenanceSource): UnverifiedIngestDisposition;
|
|
50
|
+
/**
|
|
51
|
+
* A frozen hash mismatch is benign only while its latest provenance record is an undecided, content-bound
|
|
52
|
+
* waiver created by a supported ingest path. Keep the storage/health classification centralized; downstream
|
|
53
|
+
* reuse and sync apply their own stricter allowlists before an unverified asset may leave quarantine.
|
|
54
|
+
*/
|
|
55
|
+
export declare function isActiveUnverifiedProvenance(record: ProvenanceRecord | null | undefined, frozenContentId: string | null | undefined): boolean;
|
|
56
|
+
export declare class ProvenanceWritePendingError extends Error {
|
|
57
|
+
readonly assetId: string;
|
|
58
|
+
readonly code = "PROVENANCE_WRITE_PENDING";
|
|
59
|
+
constructor(assetId: string);
|
|
10
60
|
}
|
|
11
61
|
/**
|
|
12
62
|
* Append-only JSONL sidecar (last-write-wins) at <baseDir>/provenance.jsonl. Default for an asset with NO
|
|
13
|
-
* record = trusted:
|
|
14
|
-
* write here
|
|
63
|
+
* record = trusted: verified local writers (cycleEngine self-produce and self-consistent v1 migration rows)
|
|
64
|
+
* do not write here. Every untrusted or hash-mismatched ingest path MUST mark before persistence.
|
|
15
65
|
*/
|
|
16
66
|
export declare class ProvenanceStore {
|
|
17
67
|
private readonly now;
|
|
@@ -23,22 +73,61 @@ export declare class ProvenanceStore {
|
|
|
23
73
|
private rebuildIndex;
|
|
24
74
|
private refreshUnderLock;
|
|
25
75
|
private withFreshRead;
|
|
76
|
+
private appendUnderLock;
|
|
26
77
|
/** Record provenance for an asset_id (append-only; the JSONL history is the audit trail). */
|
|
27
78
|
mark(rec: Omit<ProvenanceRecord, 'at'> & {
|
|
28
79
|
at?: string;
|
|
29
80
|
}): ProvenanceRecord;
|
|
81
|
+
/** Stage a verified Hub write without replacing an in-flight conservative marker. */
|
|
82
|
+
stageUntrustedWriteTracked(assetId: string, source: ProvenanceSource): {
|
|
83
|
+
record: ProvenanceRecord;
|
|
84
|
+
appended: boolean;
|
|
85
|
+
};
|
|
86
|
+
/** Finalize a verified write unless an operator made an explicit decision during I/O. */
|
|
87
|
+
finalizeUntrustedWrite(assetId: string, source: ProvenanceSource): ProvenanceRecord;
|
|
88
|
+
/** Stage an unverified write without overwriting an explicit trust decision or another conservative marker. */
|
|
89
|
+
stageUnverifiedWrite(assetId: string, source: ProvenanceSource, frozenContentId: string): ProvenanceRecord;
|
|
90
|
+
stageUnverifiedWriteTracked(assetId: string, source: ProvenanceSource, frozenContentId: string): {
|
|
91
|
+
record: ProvenanceRecord;
|
|
92
|
+
appended: boolean;
|
|
93
|
+
};
|
|
94
|
+
/** Atomically replace only a pending/no decision with the health-waiver reason after verified persistence. */
|
|
95
|
+
finalizeUnverifiedWrite(assetId: string, source: ProvenanceSource, reason: string, frozenContentId: string): ProvenanceRecord;
|
|
30
96
|
rollbackLast(rec: ProvenanceRecord): void;
|
|
31
97
|
get(assetId: string): ProvenanceRecord | null;
|
|
32
98
|
/** No record → trusted (local default); a record → its trusted flag. */
|
|
33
99
|
isTrusted(assetId: string): boolean;
|
|
34
100
|
/** One linearizable trust snapshot for bounded batch readers. */
|
|
35
101
|
snapshot(): ReadonlyMap<string, ProvenanceRecord>;
|
|
102
|
+
/** Compare and append one trust decision under the same cross-process lock. */
|
|
103
|
+
changeTrust(assetId: string, trusted: boolean, by: string, reason: string): ProvenanceTrustChange;
|
|
36
104
|
/** Explicit, audited untrusted→trusted promotion. Appends a new trusted record carrying who/why. */
|
|
37
105
|
promote(assetId: string, by: string, reason: string): ProvenanceRecord;
|
|
106
|
+
/** Explicit, audited trusted-to-untrusted revocation. A record-less asset is local by default. */
|
|
107
|
+
revoke(assetId: string, by: string, reason: string): ProvenanceRecord;
|
|
38
108
|
}
|
|
39
109
|
/**
|
|
40
110
|
* The sanctioned hub→local-pool landing: store the asset (store.put recomputes/normalizes the asset_id, so a
|
|
41
111
|
* remote-supplied asset_id is never trusted) and mark it untrusted in the sidecar. This is the ONLY path that
|
|
42
112
|
* should bring hub-fetched assets into the local pool — trust-first from the first byte (#30.1).
|
|
43
113
|
*/
|
|
44
|
-
export declare function ingestUntrusted(store: AssetStoreProvider, prov: ProvenanceStore, record: AssetRecord, source?: ProvenanceSource): Promise<PutResult>;
|
|
114
|
+
export declare function ingestUntrusted(store: AssetStoreProvider, prov: ProvenanceStore, record: AssetRecord, source?: ProvenanceSource): Promise<PutResult>;
|
|
115
|
+
/**
|
|
116
|
+
* Hub → local-pool landing for an asset whose content does NOT hash to its declared asset_id. The hub
|
|
117
|
+
* demonstrably rewrites delivered payloads (injected `validation`, wholesale `payload_backfill_reason`
|
|
118
|
+
* synthesis — evolver-v2#570), which breaks {@link ingestUntrusted}'s normalizeForPut self-consistency check
|
|
119
|
+
* even though the loop's own in-run reuse already consumes hub content without re-verifying it (adapter
|
|
120
|
+
* `hubReuse.ts`). Rather than hard-reject a save the operator explicitly asked for, freeze the asset under its
|
|
121
|
+
* declared (network) asset_id via `putFrozen` and mark it untrusted in the sidecar with an explicit reason.
|
|
122
|
+
* Trust-first still holds (#30.1): selection defaults to trusted-only, so an unverified asset never silently
|
|
123
|
+
* enters the reasoning pool — it lands where the operator put it and stays flagged until an explicit,
|
|
124
|
+
* audited promotion. `reason` records WHY verification was waived (e.g. hub rewrite vs synthesized payload).
|
|
125
|
+
*/
|
|
126
|
+
export declare function ingestUnverified(store: AssetStoreProvider, prov: ProvenanceStore, record: AssetRecord, reason: string, source?: ProvenanceSource): Promise<PutResult>;
|
|
127
|
+
/** Atomic frozen variant used by reuse so the logical-id check and append share one provider lock. */
|
|
128
|
+
export declare function ingestUnverifiedConditional(store: AssetStoreProvider, prov: ProvenanceStore, record: AssetRecord, reason: string, options?: ConditionalPutOptions, source?: ProvenanceSource): Promise<ConditionalPutResult>;
|
|
129
|
+
/**
|
|
130
|
+
* Conditional variant used by Hub sync to reject a logical-id collision without ever allowing a Hub record
|
|
131
|
+
* to become implicitly trusted. Providers that cannot make the condition atomically are rejected here.
|
|
132
|
+
*/
|
|
133
|
+
export declare function ingestUntrustedConditional(store: AssetStoreProvider, prov: ProvenanceStore, record: AssetRecord, options?: ConditionalPutOptions, source?: ProvenanceSource): Promise<ConditionalPutResult>;
|