@evomap/evolver-core 2.0.0-beta.0 → 2.0.0-beta.10
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/assets/gep/genes.jsonl +5 -0
- package/dist/algo/candidateAssembly.js +23 -14
- package/dist/algo/capabilityCandidates.d.ts +2 -0
- package/dist/algo/capabilityCandidates.js +5 -0
- package/dist/algo/conversationSniffer.d.ts +18 -0
- package/dist/algo/conversationSniffer.js +149 -0
- package/dist/algo/cycleEngine.d.ts +11 -0
- package/dist/algo/cycleEngine.js +14 -6
- package/dist/algo/cycleFailureClassifier.d.ts +1 -1
- package/dist/algo/cycleFailureClassifier.js +13 -5
- package/dist/algo/geneIntake.d.ts +14 -4
- package/dist/algo/geneIntake.js +37 -9
- package/dist/algo/geneSelection.d.ts +18 -1
- package/dist/algo/geneSelection.js +42 -18
- package/dist/algo/index.d.ts +2 -0
- package/dist/algo/index.js +2 -0
- package/dist/algo/memoryGraph.d.ts +62 -0
- package/dist/algo/memoryGraph.js +86 -0
- package/dist/algo/orchestrator.d.ts +3 -0
- package/dist/algo/orchestrator.js +14 -2
- package/dist/assetstore/assetSidecarRecords.d.ts +28 -0
- package/dist/assetstore/assetSidecarRecords.js +376 -0
- package/dist/assetstore/assetSidecarRecovery.d.ts +48 -0
- package/dist/assetstore/assetSidecarRecovery.js +288 -0
- package/dist/assetstore/assetStoreHealth.d.ts +75 -0
- package/dist/assetstore/assetStoreHealth.js +277 -0
- package/dist/assetstore/assetStoreLayout.d.ts +2 -0
- package/dist/assetstore/assetStoreLayout.js +6 -0
- package/dist/assetstore/assetStoreStorage.d.ts +42 -0
- package/dist/assetstore/assetStoreStorage.js +327 -0
- package/dist/assetstore/assetSyncLedger.d.ts +118 -0
- package/dist/assetstore/assetSyncLedger.js +745 -0
- package/dist/assetstore/index.d.ts +5 -1
- package/dist/assetstore/index.js +5 -1
- package/dist/assetstore/localJsonl.d.ts +12 -3
- package/dist/assetstore/localJsonl.js +131 -39
- package/dist/assetstore/pendingSignals.js +3 -1
- package/dist/assetstore/provenance.d.ts +27 -4
- package/dist/assetstore/provenance.js +107 -56
- package/dist/assetstore/provider.d.ts +26 -0
- package/dist/assetstore/provider.js +50 -0
- package/dist/assetstore/reviewFilter.js +5 -2
- package/dist/assetstore/reviewLedger.d.ts +14 -2
- package/dist/assetstore/reviewLedger.js +78 -43
- package/dist/assetstore/seedGenes.d.ts +3 -0
- package/dist/assetstore/seedGenes.js +134 -0
- package/dist/benchmark/antiGeneBenchmark.d.ts +2 -0
- package/dist/benchmark/antiGeneBenchmark.js +11 -1
- package/dist/benchmark/antiGeneImpact.d.ts +16 -0
- package/dist/benchmark/antiGeneImpact.js +34 -0
- package/dist/benchmark/antiGeneRollout.d.ts +2 -0
- package/dist/benchmark/antiGeneRollout.js +13 -1
- package/dist/benchmark/index.d.ts +2 -1
- package/dist/benchmark/index.js +2 -1
- package/dist/benchmark/triggerShift.d.ts +62 -0
- package/dist/benchmark/triggerShift.js +106 -0
- package/dist/events/eventArchive.d.ts +65 -0
- package/dist/events/eventArchive.js +343 -0
- package/dist/events/eventStore.js +2 -12
- package/dist/events/ingest.d.ts +1 -1
- package/dist/events/ingest.js +9 -0
- package/dist/events/paths.d.ts +10 -10
- package/dist/events/paths.js +20 -20
- package/dist/events/public.d.ts +3 -1
- package/dist/events/public.js +2 -1
- package/dist/events/retention.d.ts +24 -1
- package/dist/events/retention.js +133 -37
- package/dist/exec/autoExec.d.ts +6 -1
- package/dist/exec/autoExec.js +34 -0
- package/dist/exec/autonomousCycle.d.ts +2 -0
- package/dist/exec/autonomousCycle.js +5 -0
- package/dist/exec/claudeBridge.d.ts +21 -3
- package/dist/exec/claudeBridge.js +413 -26
- package/dist/exec/openPrRegistry.d.ts +8 -2
- package/dist/exec/openPrRegistry.js +32 -22
- package/dist/exec/prompt.js +9 -0
- package/dist/exec/runnerRegistry.d.ts +90 -18
- package/dist/exec/runnerRegistry.js +603 -42
- package/dist/exec/selfPrObfuscation.d.ts +2 -1
- package/dist/exec/selfPrObfuscation.js +19 -6
- package/dist/hub/agentDirectory.d.ts +90 -0
- package/dist/hub/agentDirectory.js +104 -0
- package/dist/hub/bindings.js +23 -3
- package/dist/hub/capability.d.ts +14 -2
- package/dist/hub/fake.d.ts +4 -2
- package/dist/hub/fake.js +3 -2
- package/dist/hub/index.d.ts +1 -0
- package/dist/hub/index.js +1 -0
- package/dist/hub/sanitize.js +3 -2
- package/dist/index.d.ts +2 -1
- package/dist/index.js +2 -1
- package/dist/issueReporter/index.d.ts +156 -0
- package/dist/issueReporter/index.js +1688 -0
- package/dist/mailbox/catalog.js +3 -0
- package/dist/mailbox/ipcServer.js +2 -2
- package/dist/mailbox/store.d.ts +14 -0
- package/dist/mailbox/store.js +58 -6
- package/dist/material/consumer.js +9 -6
- package/dist/material/index.d.ts +1 -0
- package/dist/material/index.js +1 -0
- package/dist/material/materialArchive.d.ts +81 -0
- package/dist/material/materialArchive.js +466 -0
- package/dist/material/materialStore.d.ts +1 -0
- package/dist/material/materialStore.js +6 -13
- package/dist/ops/savingsCore.js +1 -2
- package/dist/ops/selfUpdate.d.ts +10 -1
- package/dist/ops/selfUpdate.js +64 -15
- package/dist/personality/schema.d.ts +12 -12
- package/dist/schema/common.d.ts +1 -1
- package/dist/schema/common.js +1 -1
- package/dist/schema/material.d.ts +5 -5
- package/dist/strategy/constraintAblation.d.ts +64 -0
- package/dist/strategy/constraintAblation.js +3074 -0
- package/dist/strategy/index.d.ts +2 -1
- package/dist/strategy/index.js +2 -1
- package/dist/trace/trajectoryExport.js +2 -2
- package/dist/util/fetchPort.d.ts +1 -0
- package/dist/util/fetchPort.js +11 -0
- package/dist/util/fileLock.d.ts +24 -5
- package/dist/util/fileLock.js +288 -72
- package/dist/util/index.d.ts +1 -0
- package/dist/util/index.js +1 -0
- package/dist/wire/geneHints.d.ts +42 -1
- package/dist/wire/geneHints.js +52 -1
- package/dist/wire/index.d.ts +14 -2
- package/dist/wire/index.js +1 -1
- package/dist/workflow/dsl.d.ts +24 -3
- package/dist/workflow/dsl.js +4 -0
- package/dist/workflow/engine.d.ts +5 -1
- package/dist/workflow/engine.js +3 -0
- package/dist/workflow/index.d.ts +3 -1
- package/dist/workflow/index.js +3 -1
- package/dist/workflow/runtime.d.ts +110 -0
- package/dist/workflow/runtime.js +1298 -0
- package/dist/workflow/stateStore.d.ts +172 -0
- package/dist/workflow/stateStore.js +1044 -0
- package/package.json +6 -1
|
@@ -1,8 +1,12 @@
|
|
|
1
1
|
export * from './provider.js';
|
|
2
2
|
export * from './localJsonl.js';
|
|
3
|
+
export * from './assetStoreHealth.js';
|
|
4
|
+
export * from './assetSidecarRecovery.js';
|
|
3
5
|
export * from './remoteStub.js';
|
|
4
6
|
export * from './learningHistory.js';
|
|
5
7
|
export * from './provenance.js';
|
|
8
|
+
export * from './assetSyncLedger.js';
|
|
6
9
|
export * from './reviewLedger.js';
|
|
7
10
|
export * from './reviewFilter.js';
|
|
8
|
-
export * from './pendingSignals.js';
|
|
11
|
+
export * from './pendingSignals.js';
|
|
12
|
+
export * from './seedGenes.js';
|
package/dist/assetstore/index.js
CHANGED
|
@@ -1,8 +1,12 @@
|
|
|
1
1
|
export * from './provider.js';
|
|
2
2
|
export * from './localJsonl.js';
|
|
3
|
+
export * from './assetStoreHealth.js';
|
|
4
|
+
export * from './assetSidecarRecovery.js';
|
|
3
5
|
export * from './remoteStub.js';
|
|
4
6
|
export * from './learningHistory.js';
|
|
5
7
|
export * from './provenance.js';
|
|
8
|
+
export * from './assetSyncLedger.js';
|
|
6
9
|
export * from './reviewLedger.js';
|
|
7
10
|
export * from './reviewFilter.js';
|
|
8
|
-
export * from './pendingSignals.js';
|
|
11
|
+
export * from './pendingSignals.js';
|
|
12
|
+
export * from './seedGenes.js';
|
|
@@ -1,23 +1,32 @@
|
|
|
1
|
-
import { type AssetKind, type AssetRecord, type AssetStoreProvider, type PutResult, type SearchQuery } from './provider.js';
|
|
1
|
+
import { type AssetKind, type AssetRecord, type AssetStoreProvider, type ConditionalPutOptions, type ConditionalPutResult, type PutResult, type SearchQuery } from './provider.js';
|
|
2
2
|
/**
|
|
3
3
|
* 本地 jsonl 资产库(M3-2, 移植 v1 src/gep/assetStore.js 单写锁).
|
|
4
4
|
* 每 kind 一文件(genes/capsules/events.jsonl); append-only; O_EXCL 文件锁防并发写撕裂;
|
|
5
|
-
* 内存索引(asset_id→record)供 get/search;
|
|
5
|
+
* 内存索引(asset_id→record)供 get/search; 文件指纹变化时在共享锁内重建索引,保证多个
|
|
6
|
+
* CLI/daemon 进程之间可见; 写入也在锁内刷新后再按 asset_id 去重(内容寻址天然幂等).
|
|
6
7
|
*/
|
|
7
8
|
export declare class LocalJsonlProvider implements AssetStoreProvider {
|
|
8
9
|
readonly baseDir: string;
|
|
9
10
|
private readonly index;
|
|
10
11
|
private readonly lockPath;
|
|
12
|
+
private fileState;
|
|
11
13
|
private loaded;
|
|
12
14
|
constructor(baseDir: string);
|
|
13
|
-
private
|
|
15
|
+
private captureFileState;
|
|
16
|
+
private stateChanged;
|
|
17
|
+
private rebuildIndex;
|
|
18
|
+
private refreshUnderLock;
|
|
19
|
+
private ensureFresh;
|
|
20
|
+
private updateFileStateAfterWrite;
|
|
14
21
|
put(asset: AssetRecord): Promise<PutResult>;
|
|
22
|
+
putConditional(asset: AssetRecord, options?: ConditionalPutOptions): Promise<ConditionalPutResult>;
|
|
15
23
|
/**
|
|
16
24
|
* 迁移专用(M8-2): 以**冻结 asset_id** 原样写入, 不经 normalizeForPut 重算/校验.
|
|
17
25
|
* 仅 v1→v2 导入用(硬化 A6 存量冻结); 普通写一律走 put(). record 必须自带 asset_id.
|
|
18
26
|
*/
|
|
19
27
|
putFrozen(record: AssetRecord): Promise<PutResult>;
|
|
20
28
|
get(assetId: string): Promise<AssetRecord | null>;
|
|
29
|
+
findByLogicalId(id: string, limit?: number): Promise<AssetRecord[]>;
|
|
21
30
|
list(kind?: AssetKind, limit?: number): Promise<AssetRecord[]>;
|
|
22
31
|
search(q: SearchQuery): Promise<AssetRecord[]>;
|
|
23
32
|
/**
|
|
@@ -1,8 +1,8 @@
|
|
|
1
|
-
import { appendFileSync, existsSync, readFileSync, writeFileSync, renameSync, mkdirSync } from 'node:fs';
|
|
2
1
|
import { join } from 'node:path';
|
|
3
2
|
import { acquireLock, releaseLock } from '../util/fileLock.js';
|
|
3
|
+
import { appendUtf8Durable, assertAssetStoreDirectory, assertOptionalRegularFile, ensureAssetStoreDirectory, readUtf8Regular, regularFileFingerprint, replaceUtf8Durable, } from './assetStoreStorage.js';
|
|
4
|
+
import { LOCAL_ASSET_FILES } from './assetStoreLayout.js';
|
|
4
5
|
import { normalizeForPut, } from './provider.js';
|
|
5
|
-
const FILES = { Gene: 'genes.jsonl', Capsule: 'capsules.jsonl', EvolutionEvent: 'events.jsonl', AntiGene: 'anti-genes.jsonl' };
|
|
6
6
|
function signalsOf(a) {
|
|
7
7
|
const out = [];
|
|
8
8
|
for (const key of ['signals_match', 'signals', 'trigger', 'trigger_signals']) {
|
|
@@ -17,79 +17,157 @@ function signalsOf(a) {
|
|
|
17
17
|
/**
|
|
18
18
|
* 本地 jsonl 资产库(M3-2, 移植 v1 src/gep/assetStore.js 单写锁).
|
|
19
19
|
* 每 kind 一文件(genes/capsules/events.jsonl); append-only; O_EXCL 文件锁防并发写撕裂;
|
|
20
|
-
* 内存索引(asset_id→record)供 get/search;
|
|
20
|
+
* 内存索引(asset_id→record)供 get/search; 文件指纹变化时在共享锁内重建索引,保证多个
|
|
21
|
+
* CLI/daemon 进程之间可见; 写入也在锁内刷新后再按 asset_id 去重(内容寻址天然幂等).
|
|
21
22
|
*/
|
|
22
23
|
export class LocalJsonlProvider {
|
|
23
24
|
baseDir;
|
|
24
25
|
index = new Map();
|
|
25
26
|
lockPath;
|
|
27
|
+
fileState = new Map();
|
|
26
28
|
loaded = false;
|
|
27
29
|
// `baseDir` is public-readonly so callers that inject a store (e.g. the CLI under test) can co-locate sidecars
|
|
28
30
|
// — the ReviewLedger/ProvenanceStore — in the SAME directory, instead of defaulting to the real ~/.evomap.
|
|
29
31
|
constructor(baseDir) {
|
|
30
32
|
this.baseDir = baseDir;
|
|
31
|
-
|
|
33
|
+
ensureAssetStoreDirectory(baseDir);
|
|
32
34
|
this.lockPath = join(baseDir, '.assetstore.lock');
|
|
33
35
|
}
|
|
34
|
-
|
|
35
|
-
|
|
36
|
-
|
|
37
|
-
for (const file of Object.
|
|
36
|
+
captureFileState() {
|
|
37
|
+
assertAssetStoreDirectory(this.baseDir);
|
|
38
|
+
const state = new Map();
|
|
39
|
+
for (const [kind, file] of Object.entries(LOCAL_ASSET_FILES)) {
|
|
40
|
+
state.set(kind, regularFileFingerprint(join(this.baseDir, file)));
|
|
41
|
+
}
|
|
42
|
+
return state;
|
|
43
|
+
}
|
|
44
|
+
stateChanged(next) {
|
|
45
|
+
if (!this.loaded || next.size !== this.fileState.size)
|
|
46
|
+
return true;
|
|
47
|
+
for (const [kind, fingerprint] of next) {
|
|
48
|
+
if (this.fileState.get(kind) !== fingerprint)
|
|
49
|
+
return true;
|
|
50
|
+
}
|
|
51
|
+
return false;
|
|
52
|
+
}
|
|
53
|
+
rebuildIndex(state) {
|
|
54
|
+
const next = new Map();
|
|
55
|
+
for (const file of Object.values(LOCAL_ASSET_FILES)) {
|
|
38
56
|
const p = join(this.baseDir, file);
|
|
39
|
-
|
|
57
|
+
const raw = readUtf8Regular(p);
|
|
58
|
+
if (raw === null)
|
|
40
59
|
continue;
|
|
41
|
-
for (const line of
|
|
60
|
+
for (const line of raw.split('\n')) {
|
|
42
61
|
if (!line.trim())
|
|
43
62
|
continue;
|
|
44
63
|
try {
|
|
45
64
|
const r = JSON.parse(line);
|
|
46
65
|
if (r.asset_id)
|
|
47
|
-
|
|
66
|
+
next.set(r.asset_id, r);
|
|
48
67
|
}
|
|
49
68
|
catch { /* skip 坏行 */ }
|
|
50
69
|
}
|
|
51
70
|
}
|
|
71
|
+
this.index.clear();
|
|
72
|
+
for (const [assetId, record] of next)
|
|
73
|
+
this.index.set(assetId, record);
|
|
74
|
+
this.fileState = state;
|
|
75
|
+
this.loaded = true;
|
|
76
|
+
}
|
|
77
|
+
refreshUnderLock() {
|
|
78
|
+
const state = this.captureFileState();
|
|
79
|
+
if (this.stateChanged(state))
|
|
80
|
+
this.rebuildIndex(state);
|
|
81
|
+
}
|
|
82
|
+
ensureFresh() {
|
|
83
|
+
const state = this.captureFileState();
|
|
84
|
+
if (!this.stateChanged(state))
|
|
85
|
+
return;
|
|
86
|
+
assertOptionalRegularFile(this.lockPath, 'lock_file');
|
|
87
|
+
acquireLock(this.lockPath);
|
|
88
|
+
try {
|
|
89
|
+
this.refreshUnderLock();
|
|
90
|
+
}
|
|
91
|
+
finally {
|
|
92
|
+
releaseLock(this.lockPath);
|
|
93
|
+
}
|
|
94
|
+
}
|
|
95
|
+
updateFileStateAfterWrite() {
|
|
96
|
+
this.fileState = this.captureFileState();
|
|
52
97
|
this.loaded = true;
|
|
53
98
|
}
|
|
54
99
|
async put(asset) {
|
|
55
|
-
this.
|
|
100
|
+
return this.putConditional(asset, { allowLogicalCollision: true });
|
|
101
|
+
}
|
|
102
|
+
async putConditional(asset, options) {
|
|
56
103
|
const { record, verified } = normalizeForPut(asset);
|
|
57
|
-
|
|
58
|
-
|
|
59
|
-
|
|
104
|
+
const file = join(this.baseDir, LOCAL_ASSET_FILES[record.type]);
|
|
105
|
+
const logicalId = typeof record.id === 'string' ? record.id : undefined;
|
|
106
|
+
let collision;
|
|
107
|
+
assertOptionalRegularFile(this.lockPath, 'lock_file');
|
|
60
108
|
acquireLock(this.lockPath);
|
|
61
109
|
try {
|
|
62
|
-
//
|
|
63
|
-
|
|
64
|
-
|
|
65
|
-
|
|
110
|
+
// Refresh under the shared lock so another process cannot append between reload and dedupe.
|
|
111
|
+
this.refreshUnderLock();
|
|
112
|
+
if (this.index.has(record.asset_id)) {
|
|
113
|
+
return {
|
|
114
|
+
asset_id: record.asset_id,
|
|
115
|
+
stored: false,
|
|
116
|
+
verified,
|
|
117
|
+
status: 'already_exists',
|
|
118
|
+
logicalId,
|
|
119
|
+
};
|
|
66
120
|
}
|
|
67
|
-
|
|
68
|
-
|
|
121
|
+
collision = logicalId === undefined
|
|
122
|
+
? undefined
|
|
123
|
+
: [...this.index.values()].find((existing) => (existing.type === record.type
|
|
124
|
+
&& existing.id === logicalId
|
|
125
|
+
&& existing.asset_id !== record.asset_id));
|
|
126
|
+
if (collision && !options?.allowLogicalCollision) {
|
|
127
|
+
return {
|
|
128
|
+
asset_id: record.asset_id,
|
|
129
|
+
stored: false,
|
|
130
|
+
verified,
|
|
131
|
+
status: 'logical_collision',
|
|
132
|
+
logicalId,
|
|
133
|
+
collisionWithAssetId: collision.asset_id,
|
|
134
|
+
};
|
|
69
135
|
}
|
|
136
|
+
appendUtf8Durable(file, `${JSON.stringify(record)}\n`);
|
|
137
|
+
this.index.set(record.asset_id, record);
|
|
138
|
+
this.updateFileStateAfterWrite();
|
|
70
139
|
}
|
|
71
140
|
finally {
|
|
72
141
|
releaseLock(this.lockPath);
|
|
73
142
|
}
|
|
74
|
-
return {
|
|
143
|
+
return {
|
|
144
|
+
asset_id: record.asset_id,
|
|
145
|
+
stored: true,
|
|
146
|
+
verified,
|
|
147
|
+
status: 'stored',
|
|
148
|
+
...(collision ? {
|
|
149
|
+
logicalId,
|
|
150
|
+
collisionWithAssetId: collision.asset_id,
|
|
151
|
+
} : {}),
|
|
152
|
+
};
|
|
75
153
|
}
|
|
76
154
|
/**
|
|
77
155
|
* 迁移专用(M8-2): 以**冻结 asset_id** 原样写入, 不经 normalizeForPut 重算/校验.
|
|
78
156
|
* 仅 v1→v2 导入用(硬化 A6 存量冻结); 普通写一律走 put(). record 必须自带 asset_id.
|
|
79
157
|
*/
|
|
80
158
|
async putFrozen(record) {
|
|
81
|
-
this.ensureLoaded();
|
|
82
159
|
if (!record.asset_id)
|
|
83
160
|
throw new Error('putFrozen 需 record 自带冻结 asset_id');
|
|
84
|
-
|
|
85
|
-
|
|
86
|
-
const file = join(this.baseDir, FILES[record.type]);
|
|
161
|
+
const file = join(this.baseDir, LOCAL_ASSET_FILES[record.type]);
|
|
162
|
+
assertOptionalRegularFile(this.lockPath, 'lock_file');
|
|
87
163
|
acquireLock(this.lockPath);
|
|
88
164
|
try {
|
|
165
|
+
this.refreshUnderLock();
|
|
89
166
|
if (this.index.has(record.asset_id))
|
|
90
167
|
return { asset_id: record.asset_id, stored: false, verified: false };
|
|
91
|
-
|
|
168
|
+
appendUtf8Durable(file, `${JSON.stringify(record)}\n`);
|
|
92
169
|
this.index.set(record.asset_id, record);
|
|
170
|
+
this.updateFileStateAfterWrite();
|
|
93
171
|
}
|
|
94
172
|
finally {
|
|
95
173
|
releaseLock(this.lockPath);
|
|
@@ -97,11 +175,24 @@ export class LocalJsonlProvider {
|
|
|
97
175
|
return { asset_id: record.asset_id, stored: true, verified: false };
|
|
98
176
|
}
|
|
99
177
|
async get(assetId) {
|
|
100
|
-
this.
|
|
178
|
+
this.ensureFresh();
|
|
101
179
|
return this.index.get(assetId) ?? null;
|
|
102
180
|
}
|
|
181
|
+
async findByLogicalId(id, limit = 2) {
|
|
182
|
+
this.ensureFresh();
|
|
183
|
+
const boundedLimit = Number.isFinite(limit) ? Math.max(1, Math.min(1_000, Math.floor(limit))) : 2;
|
|
184
|
+
const out = [];
|
|
185
|
+
for (const record of this.index.values()) {
|
|
186
|
+
if (record['id'] !== id)
|
|
187
|
+
continue;
|
|
188
|
+
out.push(record);
|
|
189
|
+
if (out.length >= boundedLimit)
|
|
190
|
+
break;
|
|
191
|
+
}
|
|
192
|
+
return out;
|
|
193
|
+
}
|
|
103
194
|
async list(kind, limit = 1000) {
|
|
104
|
-
this.
|
|
195
|
+
this.ensureFresh();
|
|
105
196
|
const out = [];
|
|
106
197
|
for (const r of this.index.values()) {
|
|
107
198
|
if (!kind || r.type === kind)
|
|
@@ -112,7 +203,7 @@ export class LocalJsonlProvider {
|
|
|
112
203
|
return out;
|
|
113
204
|
}
|
|
114
205
|
async search(q) {
|
|
115
|
-
this.
|
|
206
|
+
this.ensureFresh();
|
|
116
207
|
const out = [];
|
|
117
208
|
for (const r of this.index.values()) {
|
|
118
209
|
if (q.kind && r.type !== q.kind)
|
|
@@ -125,8 +216,8 @@ export class LocalJsonlProvider {
|
|
|
125
216
|
if (q.gene && r.gene !== q.gene)
|
|
126
217
|
continue;
|
|
127
218
|
if (q.signalsAny && q.signalsAny.length > 0) {
|
|
128
|
-
const sig = new Set(signalsOf(r));
|
|
129
|
-
if (!q.signalsAny.some((
|
|
219
|
+
const sig = new Set(signalsOf(r).map((signal) => signal.trim().toLowerCase()));
|
|
220
|
+
if (!q.signalsAny.some((signal) => sig.has(signal.trim().toLowerCase())))
|
|
130
221
|
continue;
|
|
131
222
|
}
|
|
132
223
|
if (q.text) {
|
|
@@ -148,15 +239,17 @@ export class LocalJsonlProvider {
|
|
|
148
239
|
* the write lock. Returns kept/removed line counts.
|
|
149
240
|
*/
|
|
150
241
|
async compact() {
|
|
151
|
-
this.
|
|
242
|
+
assertOptionalRegularFile(this.lockPath, 'lock_file');
|
|
152
243
|
acquireLock(this.lockPath);
|
|
153
244
|
try {
|
|
245
|
+
this.refreshUnderLock();
|
|
154
246
|
let kept = 0, removed = 0;
|
|
155
|
-
for (const [kind, file] of Object.entries(
|
|
247
|
+
for (const [kind, file] of Object.entries(LOCAL_ASSET_FILES)) {
|
|
156
248
|
const p = join(this.baseDir, file);
|
|
157
|
-
|
|
249
|
+
const raw = readUtf8Regular(p);
|
|
250
|
+
if (raw === null)
|
|
158
251
|
continue;
|
|
159
|
-
const lines =
|
|
252
|
+
const lines = raw.split('\n').filter((l) => l.trim());
|
|
160
253
|
const byId = new Map(); // asset_id → raw line (last wins, matching load semantics)
|
|
161
254
|
for (const line of lines) {
|
|
162
255
|
try {
|
|
@@ -167,12 +260,11 @@ export class LocalJsonlProvider {
|
|
|
167
260
|
catch { /* drop corrupt line */ }
|
|
168
261
|
}
|
|
169
262
|
const out = byId.size ? `${[...byId.values()].join('\n')}\n` : '';
|
|
170
|
-
|
|
171
|
-
writeFileSync(tmp, out);
|
|
172
|
-
renameSync(tmp, p); // atomic replace; a crash mid-compact leaves the original intact
|
|
263
|
+
replaceUtf8Durable(p, out);
|
|
173
264
|
kept += byId.size;
|
|
174
265
|
removed += lines.length - byId.size;
|
|
175
266
|
}
|
|
267
|
+
this.rebuildIndex(this.captureFileState());
|
|
176
268
|
return { kept, removed };
|
|
177
269
|
}
|
|
178
270
|
finally {
|
|
@@ -1,5 +1,5 @@
|
|
|
1
1
|
import { existsSync, mkdirSync, readFileSync, renameSync, statSync, writeFileSync } from 'node:fs';
|
|
2
|
-
import { dirname, join, resolve } from 'node:path';
|
|
2
|
+
import { basename, dirname, join, resolve } from 'node:path';
|
|
3
3
|
import { acquireLock, releaseLock } from '../util/fileLock.js';
|
|
4
4
|
import { LocalJsonlProvider } from './localJsonl.js';
|
|
5
5
|
const MAX_SIGNAL_LENGTH = 200;
|
|
@@ -33,6 +33,8 @@ function findRepoRoot(start) {
|
|
|
33
33
|
while (current) {
|
|
34
34
|
if (existsSync(join(current, '.git')))
|
|
35
35
|
return current;
|
|
36
|
+
if (basename(current) === 'node_modules')
|
|
37
|
+
return null;
|
|
36
38
|
const parent = dirname(current);
|
|
37
39
|
if (parent === current)
|
|
38
40
|
return null;
|
|
@@ -1,13 +1,21 @@
|
|
|
1
|
-
import { type AssetStoreProvider, type AssetRecord, type PutResult } from './provider.js';
|
|
1
|
+
import { type AssetStoreProvider, type AssetRecord, type ConditionalPutOptions, type ConditionalPutResult, type PutResult } from './provider.js';
|
|
2
2
|
export type ProvenanceSource = 'local' | 'migrated' | 'hub';
|
|
3
|
+
export type ProvenanceDecision = 'promoted' | 'revoked';
|
|
3
4
|
export interface ProvenanceRecord {
|
|
4
5
|
assetId: string;
|
|
5
6
|
source: ProvenanceSource;
|
|
6
7
|
trusted: boolean;
|
|
7
8
|
at: string;
|
|
9
|
+
decision?: ProvenanceDecision;
|
|
10
|
+
decidedBy?: string;
|
|
11
|
+
/** Legacy promotion actor field kept for existing sidecar readers. */
|
|
8
12
|
promotedBy?: string;
|
|
9
13
|
reason?: string;
|
|
10
14
|
}
|
|
15
|
+
export interface ProvenanceTrustChange {
|
|
16
|
+
changed: boolean;
|
|
17
|
+
record: ProvenanceRecord;
|
|
18
|
+
}
|
|
11
19
|
/**
|
|
12
20
|
* Append-only JSONL sidecar (last-write-wins) at <baseDir>/provenance.jsonl. Default for an asset with NO
|
|
13
21
|
* record = trusted: the only local writers (cycleEngine self-produce, v1 migration) are trusted and never
|
|
@@ -16,10 +24,14 @@ export interface ProvenanceRecord {
|
|
|
16
24
|
export declare class ProvenanceStore {
|
|
17
25
|
private readonly now;
|
|
18
26
|
private readonly path;
|
|
27
|
+
private readonly lockPath;
|
|
19
28
|
private readonly index;
|
|
20
|
-
private
|
|
29
|
+
private fileState;
|
|
21
30
|
constructor(baseDir: string, now?: () => number);
|
|
22
|
-
private
|
|
31
|
+
private rebuildIndex;
|
|
32
|
+
private refreshUnderLock;
|
|
33
|
+
private withFreshRead;
|
|
34
|
+
private appendUnderLock;
|
|
23
35
|
/** Record provenance for an asset_id (append-only; the JSONL history is the audit trail). */
|
|
24
36
|
mark(rec: Omit<ProvenanceRecord, 'at'> & {
|
|
25
37
|
at?: string;
|
|
@@ -28,12 +40,23 @@ export declare class ProvenanceStore {
|
|
|
28
40
|
get(assetId: string): ProvenanceRecord | null;
|
|
29
41
|
/** No record → trusted (local default); a record → its trusted flag. */
|
|
30
42
|
isTrusted(assetId: string): boolean;
|
|
43
|
+
/** One linearizable trust snapshot for bounded batch readers. */
|
|
44
|
+
snapshot(): ReadonlyMap<string, ProvenanceRecord>;
|
|
45
|
+
/** Compare and append one trust decision under the same cross-process lock. */
|
|
46
|
+
changeTrust(assetId: string, trusted: boolean, by: string, reason: string): ProvenanceTrustChange;
|
|
31
47
|
/** Explicit, audited untrusted→trusted promotion. Appends a new trusted record carrying who/why. */
|
|
32
48
|
promote(assetId: string, by: string, reason: string): ProvenanceRecord;
|
|
49
|
+
/** Explicit, audited trusted-to-untrusted revocation. A record-less asset is local by default. */
|
|
50
|
+
revoke(assetId: string, by: string, reason: string): ProvenanceRecord;
|
|
33
51
|
}
|
|
34
52
|
/**
|
|
35
53
|
* The sanctioned hub→local-pool landing: store the asset (store.put recomputes/normalizes the asset_id, so a
|
|
36
54
|
* remote-supplied asset_id is never trusted) and mark it untrusted in the sidecar. This is the ONLY path that
|
|
37
55
|
* should bring hub-fetched assets into the local pool — trust-first from the first byte (#30.1).
|
|
38
56
|
*/
|
|
39
|
-
export declare function ingestUntrusted(store: AssetStoreProvider, prov: ProvenanceStore, record: AssetRecord, source?: ProvenanceSource): Promise<PutResult>;
|
|
57
|
+
export declare function ingestUntrusted(store: AssetStoreProvider, prov: ProvenanceStore, record: AssetRecord, source?: ProvenanceSource): Promise<PutResult>;
|
|
58
|
+
/**
|
|
59
|
+
* Conditional variant used by Hub sync to reject a logical-id collision without ever allowing a Hub record
|
|
60
|
+
* to become implicitly trusted. Providers that cannot make the condition atomically are rejected here.
|
|
61
|
+
*/
|
|
62
|
+
export declare function ingestUntrustedConditional(store: AssetStoreProvider, prov: ProvenanceStore, record: AssetRecord, options?: ConditionalPutOptions, source?: ProvenanceSource): Promise<ConditionalPutResult>;
|
|
@@ -3,9 +3,13 @@
|
|
|
3
3
|
// by asset_id, NOT a field on the asset: asset_id = sha256(canonicalize(asset)), and "how we got it" metadata
|
|
4
4
|
// must not enter the content hash (#30.2), or it would break content-addressing. Trust-first by construction:
|
|
5
5
|
// selection defaults to trusted-only; an untrusted asset is promoted to trusted only by an explicit, logged act.
|
|
6
|
-
import { appendFileSync, closeSync, existsSync, openSync, readFileSync, readSync, mkdirSync, statSync, truncateSync } from 'node:fs';
|
|
7
6
|
import { join, dirname } from 'node:path';
|
|
8
|
-
import { normalizeForPut } from './provider.js';
|
|
7
|
+
import { normalizeForPut, supportsAtomicConditionalPut, validateConditionalPutResult, } from './provider.js';
|
|
8
|
+
import { appendUtf8Durable, assertAssetStoreDirectory, ensureAssetStoreDirectory, readUtf8Regular, regularFileFingerprint, truncateUtf8SuffixDurable, withAssetStoreLock, } from './assetStoreStorage.js';
|
|
9
|
+
import { assertTrustSidecarHealthy, parseProvenanceRecord, parseSidecarJsonl, } from './assetSidecarRecords.js';
|
|
10
|
+
function immutableRecord(record) {
|
|
11
|
+
return Object.freeze({ ...record });
|
|
12
|
+
}
|
|
9
13
|
/**
|
|
10
14
|
* Append-only JSONL sidecar (last-write-wins) at <baseDir>/provenance.jsonl. Default for an asset with NO
|
|
11
15
|
* record = trusted: the only local writers (cycleEngine self-produce, v1 migration) are trusted and never
|
|
@@ -14,81 +18,113 @@ import { normalizeForPut } from './provider.js';
|
|
|
14
18
|
export class ProvenanceStore {
|
|
15
19
|
now;
|
|
16
20
|
path;
|
|
21
|
+
lockPath;
|
|
17
22
|
index = new Map();
|
|
18
|
-
|
|
23
|
+
fileState = null;
|
|
19
24
|
constructor(baseDir, now = Date.now) {
|
|
20
25
|
this.now = now;
|
|
26
|
+
ensureAssetStoreDirectory(baseDir);
|
|
21
27
|
this.path = join(baseDir, 'provenance.jsonl');
|
|
28
|
+
this.lockPath = join(baseDir, '.assetstore.lock');
|
|
22
29
|
}
|
|
23
|
-
|
|
24
|
-
|
|
25
|
-
|
|
26
|
-
if (
|
|
27
|
-
|
|
28
|
-
|
|
29
|
-
|
|
30
|
-
|
|
31
|
-
const r = JSON.parse(line);
|
|
32
|
-
if (r.assetId)
|
|
33
|
-
this.index.set(r.assetId, r);
|
|
34
|
-
}
|
|
35
|
-
catch { /* skip corrupt line */ }
|
|
36
|
-
}
|
|
30
|
+
rebuildIndex(state) {
|
|
31
|
+
const next = new Map();
|
|
32
|
+
const raw = state === 'missing' ? null : readUtf8Regular(this.path);
|
|
33
|
+
if (raw !== null) {
|
|
34
|
+
const parsed = parseSidecarJsonl(raw, parseProvenanceRecord);
|
|
35
|
+
assertTrustSidecarHealthy('provenance', parsed);
|
|
36
|
+
for (const record of parsed.records)
|
|
37
|
+
next.set(record.assetId, immutableRecord(record));
|
|
37
38
|
}
|
|
38
|
-
this.
|
|
39
|
+
this.index.clear();
|
|
40
|
+
for (const [assetId, record] of next)
|
|
41
|
+
this.index.set(assetId, record);
|
|
42
|
+
this.fileState = state;
|
|
43
|
+
}
|
|
44
|
+
refreshUnderLock() {
|
|
45
|
+
const state = regularFileFingerprint(this.path);
|
|
46
|
+
if (state !== this.fileState)
|
|
47
|
+
this.rebuildIndex(state);
|
|
48
|
+
}
|
|
49
|
+
withFreshRead(read) {
|
|
50
|
+
assertAssetStoreDirectory(dirname(this.path));
|
|
51
|
+
return withAssetStoreLock(this.lockPath, () => {
|
|
52
|
+
this.refreshUnderLock();
|
|
53
|
+
return read(this.index);
|
|
54
|
+
});
|
|
55
|
+
}
|
|
56
|
+
appendUnderLock(full) {
|
|
57
|
+
const stored = immutableRecord(full);
|
|
58
|
+
appendUtf8Durable(this.path, `${JSON.stringify(stored)}\n`);
|
|
59
|
+
this.index.set(stored.assetId, stored);
|
|
60
|
+
this.fileState = regularFileFingerprint(this.path);
|
|
61
|
+
return stored;
|
|
39
62
|
}
|
|
40
63
|
/** Record provenance for an asset_id (append-only; the JSONL history is the audit trail). */
|
|
41
64
|
mark(rec) {
|
|
42
|
-
this.load();
|
|
43
65
|
const full = { ...rec, at: rec.at ?? new Date(this.now()).toISOString() };
|
|
44
|
-
|
|
45
|
-
|
|
46
|
-
|
|
47
|
-
|
|
66
|
+
assertAssetStoreDirectory(dirname(this.path));
|
|
67
|
+
return withAssetStoreLock(this.lockPath, () => {
|
|
68
|
+
this.refreshUnderLock();
|
|
69
|
+
return this.appendUnderLock(full);
|
|
70
|
+
});
|
|
48
71
|
}
|
|
49
72
|
rollbackLast(rec) {
|
|
50
73
|
const line = `${JSON.stringify(rec)}\n`;
|
|
51
|
-
const lineBytes = Buffer.byteLength(line, 'utf8');
|
|
52
74
|
try {
|
|
53
|
-
|
|
54
|
-
|
|
55
|
-
|
|
56
|
-
|
|
57
|
-
|
|
58
|
-
|
|
59
|
-
|
|
60
|
-
|
|
61
|
-
try {
|
|
62
|
-
readSync(fd, buf, 0, lineBytes, offset);
|
|
63
|
-
}
|
|
64
|
-
finally {
|
|
65
|
-
closeSync(fd);
|
|
66
|
-
}
|
|
67
|
-
if (buf.toString('utf8') !== line)
|
|
68
|
-
return;
|
|
69
|
-
truncateSync(this.path, offset);
|
|
70
|
-
this.index.clear();
|
|
71
|
-
this.loaded = false;
|
|
75
|
+
assertAssetStoreDirectory(dirname(this.path));
|
|
76
|
+
withAssetStoreLock(this.lockPath, () => {
|
|
77
|
+
if (!truncateUtf8SuffixDurable(this.path, line)) {
|
|
78
|
+
this.refreshUnderLock();
|
|
79
|
+
return;
|
|
80
|
+
}
|
|
81
|
+
this.rebuildIndex(regularFileFingerprint(this.path));
|
|
82
|
+
});
|
|
72
83
|
}
|
|
73
84
|
catch {
|
|
74
85
|
this.index.clear();
|
|
75
|
-
this.
|
|
86
|
+
this.fileState = null;
|
|
76
87
|
}
|
|
77
88
|
}
|
|
78
89
|
get(assetId) {
|
|
79
|
-
this.
|
|
80
|
-
return this.index.get(assetId) ?? null;
|
|
90
|
+
return this.withFreshRead((index) => index.get(assetId) ?? null);
|
|
81
91
|
}
|
|
82
92
|
/** No record → trusted (local default); a record → its trusted flag. */
|
|
83
93
|
isTrusted(assetId) {
|
|
84
|
-
this.
|
|
85
|
-
|
|
86
|
-
|
|
94
|
+
return this.withFreshRead((index) => index.get(assetId)?.trusted ?? true);
|
|
95
|
+
}
|
|
96
|
+
/** One linearizable trust snapshot for bounded batch readers. */
|
|
97
|
+
snapshot() {
|
|
98
|
+
return this.withFreshRead((index) => new Map(index));
|
|
99
|
+
}
|
|
100
|
+
/** Compare and append one trust decision under the same cross-process lock. */
|
|
101
|
+
changeTrust(assetId, trusted, by, reason) {
|
|
102
|
+
assertAssetStoreDirectory(dirname(this.path));
|
|
103
|
+
return withAssetStoreLock(this.lockPath, () => {
|
|
104
|
+
this.refreshUnderLock();
|
|
105
|
+
const current = this.index.get(assetId) ?? null;
|
|
106
|
+
if (current?.trusted === trusted)
|
|
107
|
+
return { changed: false, record: current };
|
|
108
|
+
const full = {
|
|
109
|
+
assetId,
|
|
110
|
+
source: current?.source ?? 'local',
|
|
111
|
+
trusted,
|
|
112
|
+
at: new Date(this.now()).toISOString(),
|
|
113
|
+
decision: trusted ? 'promoted' : 'revoked',
|
|
114
|
+
decidedBy: by,
|
|
115
|
+
...(trusted ? { promotedBy: by } : {}),
|
|
116
|
+
reason,
|
|
117
|
+
};
|
|
118
|
+
return { changed: true, record: this.appendUnderLock(full) };
|
|
119
|
+
});
|
|
87
120
|
}
|
|
88
121
|
/** Explicit, audited untrusted→trusted promotion. Appends a new trusted record carrying who/why. */
|
|
89
122
|
promote(assetId, by, reason) {
|
|
90
|
-
|
|
91
|
-
|
|
123
|
+
return this.changeTrust(assetId, true, by, reason).record;
|
|
124
|
+
}
|
|
125
|
+
/** Explicit, audited trusted-to-untrusted revocation. A record-less asset is local by default. */
|
|
126
|
+
revoke(assetId, by, reason) {
|
|
127
|
+
return this.changeTrust(assetId, false, by, reason).record;
|
|
92
128
|
}
|
|
93
129
|
}
|
|
94
130
|
/**
|
|
@@ -99,11 +135,26 @@ export class ProvenanceStore {
|
|
|
99
135
|
export async function ingestUntrusted(store, prov, record, source = 'hub') {
|
|
100
136
|
const normalized = normalizeForPut(record);
|
|
101
137
|
const mark = prov.mark({ assetId: normalized.record.asset_id, source, trusted: false });
|
|
102
|
-
|
|
103
|
-
|
|
104
|
-
|
|
105
|
-
|
|
138
|
+
const result = await store.put(record);
|
|
139
|
+
// A thrown write has an ambiguous outcome: the asset may have reached disk before the acknowledgement was
|
|
140
|
+
// lost. Keep the untrusted marker in that case so a persisted Hub asset can never fall through the default
|
|
141
|
+
// no-record => trusted policy. Only an explicit no-write result is safe to roll back.
|
|
142
|
+
if (!result.stored)
|
|
106
143
|
prov.rollbackLast(mark);
|
|
107
|
-
|
|
144
|
+
return result;
|
|
145
|
+
}
|
|
146
|
+
/**
|
|
147
|
+
* Conditional variant used by Hub sync to reject a logical-id collision without ever allowing a Hub record
|
|
148
|
+
* to become implicitly trusted. Providers that cannot make the condition atomically are rejected here.
|
|
149
|
+
*/
|
|
150
|
+
export async function ingestUntrustedConditional(store, prov, record, options, source = 'hub') {
|
|
151
|
+
if (!supportsAtomicConditionalPut(store)) {
|
|
152
|
+
throw new Error('asset store does not support conditional writes');
|
|
108
153
|
}
|
|
154
|
+
const normalized = normalizeForPut(record);
|
|
155
|
+
const mark = prov.mark({ assetId: normalized.record.asset_id, source, trusted: false });
|
|
156
|
+
const result = validateConditionalPutResult(await store.putConditional(record, options), normalized.record.asset_id, options);
|
|
157
|
+
if (!result.stored)
|
|
158
|
+
prov.rollbackLast(mark);
|
|
159
|
+
return result;
|
|
109
160
|
}
|