@evomap/evolver-core 2.0.0-beta.2 → 2.0.0-beta.22
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/assets/gep/genes.jsonl +5 -5
- package/dist/algo/bans.d.ts +10 -1
- package/dist/algo/bans.js +56 -6
- package/dist/algo/candidateAssembly.d.ts +25 -2
- package/dist/algo/candidateAssembly.js +171 -26
- package/dist/algo/capabilityCandidates.js +10 -0
- package/dist/algo/conversationSniffer.js +25 -8
- package/dist/algo/cycleEngine.d.ts +63 -5
- package/dist/algo/cycleEngine.js +320 -42
- package/dist/algo/cycleFailureClassifier.d.ts +1 -1
- package/dist/algo/evolutionEvent.d.ts +26 -0
- package/dist/algo/evolutionEvent.js +32 -0
- package/dist/algo/exploration.d.ts +7 -0
- package/dist/algo/exploration.js +16 -3
- package/dist/algo/geneHealth.d.ts +36 -3
- package/dist/algo/geneHealth.js +47 -4
- package/dist/algo/geneIntake.d.ts +46 -1
- package/dist/algo/geneIntake.js +121 -10
- package/dist/algo/geneSelection.d.ts +126 -6
- package/dist/algo/geneSelection.js +479 -36
- package/dist/algo/index.d.ts +6 -1
- package/dist/algo/index.js +6 -1
- package/dist/algo/kautoProjection.d.ts +41 -0
- package/dist/algo/kautoProjection.js +95 -0
- package/dist/algo/kautoValidator.d.ts +68 -0
- package/dist/algo/kautoValidator.js +256 -0
- package/dist/algo/memoryGraph.d.ts +62 -0
- package/dist/algo/memoryGraph.js +86 -0
- package/dist/algo/orchestrator.d.ts +17 -1
- package/dist/algo/orchestrator.js +30 -4
- package/dist/algo/publishEligibility.d.ts +34 -0
- package/dist/algo/publishEligibility.js +52 -0
- package/dist/algo/solidify.d.ts +11 -2
- package/dist/algo/solidify.js +37 -7
- package/dist/algo/ucb1.d.ts +53 -0
- package/dist/algo/ucb1.js +156 -0
- package/dist/assetrepair/hubRejection.d.ts +12 -0
- package/dist/assetrepair/hubRejection.js +109 -0
- package/dist/assetrepair/index.d.ts +2 -0
- package/dist/assetrepair/index.js +2 -0
- package/dist/assetrepair/repair.d.ts +33 -0
- package/dist/assetrepair/repair.js +155 -0
- package/dist/assetstore/assetSidecarRecords.d.ts +28 -0
- package/dist/assetstore/assetSidecarRecords.js +384 -0
- package/dist/assetstore/assetSidecarRecovery.d.ts +48 -0
- package/dist/assetstore/assetSidecarRecovery.js +288 -0
- package/dist/assetstore/assetStoreHealth.d.ts +81 -0
- package/dist/assetstore/assetStoreHealth.js +319 -0
- package/dist/assetstore/assetStoreLayout.d.ts +2 -0
- package/dist/assetstore/assetStoreLayout.js +6 -0
- package/dist/assetstore/assetStoreStorage.d.ts +42 -0
- package/dist/assetstore/assetStoreStorage.js +336 -0
- package/dist/assetstore/assetSyncLedger.d.ts +91 -1
- package/dist/assetstore/assetSyncLedger.js +718 -59
- package/dist/assetstore/foreignJsonlSource.d.ts +48 -0
- package/dist/assetstore/foreignJsonlSource.js +150 -0
- package/dist/assetstore/index.d.ts +5 -0
- package/dist/assetstore/index.js +5 -0
- package/dist/assetstore/learningHistory.js +3 -3
- package/dist/assetstore/localAssetStoreSnapshot.d.ts +51 -0
- package/dist/assetstore/localAssetStoreSnapshot.js +329 -0
- package/dist/assetstore/localJsonl.d.ts +12 -1
- package/dist/assetstore/localJsonl.js +251 -42
- package/dist/assetstore/provenance.d.ts +93 -4
- package/dist/assetstore/provenance.js +368 -84
- package/dist/assetstore/provider.d.ts +63 -0
- package/dist/assetstore/provider.js +97 -6
- package/dist/assetstore/reviewFilter.d.ts +19 -1
- package/dist/assetstore/reviewFilter.js +39 -1
- package/dist/assetstore/reviewLedger.d.ts +8 -2
- package/dist/assetstore/reviewLedger.js +71 -45
- package/dist/assetstore/unionReadStore.d.ts +25 -0
- package/dist/assetstore/unionReadStore.js +119 -0
- package/dist/benchmark/antiGeneBenchmark.d.ts +2 -0
- package/dist/benchmark/antiGeneBenchmark.js +4 -3
- package/dist/benchmark/antiGeneRollout.d.ts +2 -0
- package/dist/benchmark/antiGeneRollout.js +4 -3
- package/dist/benchmark/index.d.ts +3 -1
- package/dist/benchmark/index.js +3 -1
- package/dist/benchmark/selectionFlatAbstention.d.ts +152 -0
- package/dist/benchmark/selectionFlatAbstention.js +481 -0
- package/dist/benchmark/triggerShift.d.ts +62 -0
- package/dist/benchmark/triggerShift.js +106 -0
- package/dist/bootstrap/envFingerprint.d.ts +9 -0
- package/dist/bootstrap/envFingerprint.js +5 -0
- package/dist/bootstrap/index.d.ts +3 -1
- package/dist/bootstrap/index.js +3 -1
- package/dist/bootstrap/lifecycleBootstrap.d.ts +111 -0
- package/dist/bootstrap/lifecycleBootstrap.js +433 -0
- package/dist/bootstrap/v1EnvCompat.d.ts +113 -0
- package/dist/bootstrap/v1EnvCompat.js +303 -0
- package/dist/events/eventArchive.d.ts +2 -0
- package/dist/events/eventArchive.js +13 -3
- package/dist/events/eventSchema.d.ts +7 -7
- package/dist/events/eventStore.d.ts +2 -0
- package/dist/events/eventStore.js +5 -1
- package/dist/events/ingest.d.ts +2 -1
- package/dist/events/ingest.js +15 -0
- package/dist/events/paths.d.ts +12 -10
- package/dist/events/paths.js +24 -20
- package/dist/events/public.d.ts +2 -2
- package/dist/events/public.js +2 -2
- package/dist/events/reports.d.ts +2 -0
- package/dist/events/reports.js +4 -0
- package/dist/exec/autoExec.d.ts +68 -4
- package/dist/exec/autoExec.js +392 -33
- package/dist/exec/autonomousCycle.d.ts +31 -4
- package/dist/exec/autonomousCycle.js +71 -13
- package/dist/exec/claudeBridge.d.ts +80 -15
- package/dist/exec/claudeBridge.js +845 -76
- package/dist/exec/executionBinding.d.ts +414 -0
- package/dist/exec/executionBinding.js +588 -0
- package/dist/exec/index.d.ts +1 -0
- package/dist/exec/index.js +1 -0
- package/dist/exec/openPrRegistry.d.ts +8 -2
- package/dist/exec/openPrRegistry.js +32 -22
- package/dist/exec/prompt.js +14 -1
- package/dist/exec/proofOfWork.d.ts +1 -1
- package/dist/exec/proofOfWork.js +2 -2
- package/dist/exec/runnerRegistry.d.ts +153 -36
- package/dist/exec/runnerRegistry.js +848 -65
- package/dist/exec/selfPr.js +1 -7
- package/dist/feedback/envelope.d.ts +61 -0
- package/dist/feedback/envelope.js +168 -0
- package/dist/feedback/index.d.ts +1 -0
- package/dist/feedback/index.js +1 -0
- package/dist/hooks/hooks.js +1 -0
- package/dist/hub/assetCallLog.d.ts +35 -1
- package/dist/hub/assetCallLog.js +124 -1
- package/dist/hub/bindings.d.ts +8 -1
- package/dist/hub/bindings.js +29 -8
- package/dist/hub/capability.d.ts +130 -4
- package/dist/hub/conversationDistiller.d.ts +19 -0
- package/dist/hub/conversationDistiller.js +115 -37
- package/dist/hub/fake.d.ts +3 -2
- package/dist/hub/fake.js +2 -1
- package/dist/hub/index.d.ts +1 -0
- package/dist/hub/index.js +1 -0
- package/dist/hub/questionGenerator.d.ts +5 -1
- package/dist/hub/questionGenerator.js +8 -6
- package/dist/hub/recipeCompose.d.ts +27 -0
- package/dist/hub/recipeCompose.js +90 -0
- package/dist/hub/sanitize.js +122 -7
- package/dist/index.d.ts +6 -1
- package/dist/index.js +7 -1
- package/dist/issueReporter/index.d.ts +156 -0
- package/dist/issueReporter/index.js +1688 -0
- package/dist/mailbox/dispatch.d.ts +1 -1
- package/dist/mailbox/dispatch.js +22 -6
- package/dist/mailbox/envelope.d.ts +7 -1
- package/dist/mailbox/envelope.js +9 -2
- package/dist/mailbox/ipcServer.d.ts +12 -2
- package/dist/mailbox/ipcServer.js +183 -13
- package/dist/mailbox/store.d.ts +89 -3
- package/dist/mailbox/store.js +895 -41
- package/dist/modelCompatibility.d.ts +164 -0
- package/dist/modelCompatibility.js +309 -0
- package/dist/observers/valueDigestObserver.d.ts +9 -0
- package/dist/observers/valueDigestObserver.js +35 -2
- package/dist/ops/cleanup.js +1 -1
- package/dist/ops/evolutionGraphProjection.d.ts +20 -0
- package/dist/ops/evolutionGraphProjection.js +315 -0
- package/dist/ops/index.d.ts +2 -1
- package/dist/ops/index.js +2 -1
- package/dist/ops/savingsCore.js +1 -2
- package/dist/ops/selfUpdate.d.ts +18 -1
- package/dist/ops/selfUpdate.js +88 -23
- package/dist/ops/valueOutreach.d.ts +3 -1
- package/dist/ops/valueOutreach.js +5 -1
- package/dist/personality/schema.d.ts +24 -24
- package/dist/schema/evolutionGraph.d.ts +784 -0
- package/dist/schema/evolutionGraph.js +187 -0
- package/dist/schema/index.d.ts +1 -0
- package/dist/schema/index.js +1 -0
- package/dist/schema/proofOfWork.d.ts +125 -6
- package/dist/schema/proofOfWork.js +102 -4
- package/dist/schema/signal.d.ts +3 -3
- package/dist/schema/signal.js +1 -1
- package/dist/shadow/shadowHub.js +1 -0
- package/dist/signals/curriculum.d.ts +55 -0
- package/dist/signals/curriculum.js +202 -0
- package/dist/signals/cycleHistoryFromEvents.js +17 -8
- package/dist/signals/expand.d.ts +15 -1
- package/dist/signals/expand.js +169 -1
- package/dist/signals/extractor.d.ts +2 -2
- package/dist/signals/extractor.js +31 -6
- package/dist/signals/index.d.ts +4 -1
- package/dist/signals/index.js +4 -1
- package/dist/signals/metaSignals.d.ts +4 -0
- package/dist/signals/metaSignals.js +42 -0
- package/dist/signals/scopeVocabulary.d.ts +75 -0
- package/dist/signals/scopeVocabulary.js +91 -0
- package/dist/signals/signalGate.js +1 -1
- package/dist/signals/taskDomain.d.ts +22 -0
- package/dist/signals/taskDomain.js +43 -0
- package/dist/strategy/constraintAblation.d.ts +64 -0
- package/dist/strategy/constraintAblation.js +2820 -0
- package/dist/strategy/constraintAblationPredicates.d.ts +31 -0
- package/dist/strategy/constraintAblationPredicates.js +339 -0
- package/dist/strategy/index.d.ts +2 -1
- package/dist/strategy/index.js +2 -1
- package/dist/trace/index.d.ts +3 -1
- package/dist/trace/index.js +3 -1
- package/dist/trace/learningTrace.d.ts +216 -0
- package/dist/trace/learningTrace.js +298 -0
- package/dist/trace/proxyTurns.d.ts +31 -0
- package/dist/trace/proxyTurns.js +137 -0
- package/dist/trace/trajectory.d.ts +8 -0
- package/dist/trace/trajectory.js +14 -2
- package/dist/util/fetchPort.d.ts +1 -0
- package/dist/util/fetchPort.js +11 -0
- package/dist/util/fileLock.d.ts +113 -7
- package/dist/util/fileLock.js +1035 -80
- package/dist/util/index.d.ts +3 -1
- package/dist/util/index.js +2 -1
- package/dist/verify/index.d.ts +2 -1
- package/dist/verify/index.js +1 -1
- package/dist/verify/sandboxRunner.d.ts +30 -0
- package/dist/verify/sandboxRunner.js +370 -27
- package/dist/verify/sandboxedValidation.d.ts +16 -2
- package/dist/verify/sandboxedValidation.js +200 -22
- package/dist/verify/validation.d.ts +35 -4
- package/dist/verify/validation.js +164 -16
- package/dist/wire/geneHints.d.ts +88 -16
- package/dist/wire/geneHints.js +124 -15
- package/dist/wire/index.d.ts +8 -3
- package/dist/wire/index.js +2 -2
- package/dist/wire/schemaGate.d.ts +21 -0
- package/dist/wire/schemaGate.js +119 -10
- package/dist/workflow/dsl.d.ts +24 -3
- package/dist/workflow/dsl.js +4 -0
- package/dist/workflow/engine.d.ts +5 -1
- package/dist/workflow/engine.js +3 -0
- package/dist/workflow/index.d.ts +3 -1
- package/dist/workflow/index.js +3 -1
- package/dist/workflow/runtime.d.ts +110 -0
- package/dist/workflow/runtime.js +1298 -0
- package/dist/workflow/stateStore.d.ts +172 -0
- package/dist/workflow/stateStore.js +1044 -0
- package/package.json +12 -4
|
@@ -0,0 +1,288 @@
|
|
|
1
|
+
import { createHash } from 'node:crypto';
|
|
2
|
+
import { existsSync, mkdirSync } from 'node:fs';
|
|
3
|
+
import { join } from 'node:path';
|
|
4
|
+
import { TextDecoder } from 'node:util';
|
|
5
|
+
import { canonicalize } from '../wire/index.js';
|
|
6
|
+
import { LockReleaseError } from '../util/fileLock.js';
|
|
7
|
+
import { AssetStoreReadLimitError, assertAssetStoreDirectory, createBufferDurableExclusive, fsyncDirectoryBestEffort, readRegularBuffer, regularFileFingerprint, replaceUtf8Durable, withAssetStoreLock, } from './assetStoreStorage.js';
|
|
8
|
+
import { parseAssetSyncSidecarRecord, parseProvenanceRecord, parseReviewRecord, } from './assetSidecarRecords.js';
|
|
9
|
+
export class AssetSidecarRecoveryError extends Error {
|
|
10
|
+
reason;
|
|
11
|
+
code = 'ASSET_SIDECAR_RECOVERY_FAILED';
|
|
12
|
+
constructor(reason) {
|
|
13
|
+
super(`asset sidecar recovery failed: ${reason}`);
|
|
14
|
+
this.reason = reason;
|
|
15
|
+
this.name = 'AssetSidecarRecoveryError';
|
|
16
|
+
}
|
|
17
|
+
}
|
|
18
|
+
export const DEFAULT_SIDECAR_RECOVERY_MAX_FILE_BYTES = 64 * 1024 * 1024;
|
|
19
|
+
function recoveryFileLimit(value) {
|
|
20
|
+
if (value === undefined || !Number.isFinite(value))
|
|
21
|
+
return DEFAULT_SIDECAR_RECOVERY_MAX_FILE_BYTES;
|
|
22
|
+
return Math.min(DEFAULT_SIDECAR_RECOVERY_MAX_FILE_BYTES, Math.max(1, Math.floor(value)));
|
|
23
|
+
}
|
|
24
|
+
const SIDECAR_FILES = {
|
|
25
|
+
provenance: 'provenance.jsonl',
|
|
26
|
+
review: 'review.jsonl',
|
|
27
|
+
'asset-sync': 'asset-sync.jsonl',
|
|
28
|
+
};
|
|
29
|
+
const RECORD_PARSERS = {
|
|
30
|
+
provenance: parseProvenanceRecord,
|
|
31
|
+
review: parseReviewRecord,
|
|
32
|
+
'asset-sync': parseAssetSyncSidecarRecord,
|
|
33
|
+
};
|
|
34
|
+
function isErrno(error, code) {
|
|
35
|
+
return typeof error === 'object' && error !== null && error.code === code;
|
|
36
|
+
}
|
|
37
|
+
function digest(bytes) {
|
|
38
|
+
return createHash('sha256').update(bytes).digest('hex');
|
|
39
|
+
}
|
|
40
|
+
function readBounded(path, limit, role) {
|
|
41
|
+
try {
|
|
42
|
+
const before = regularFileFingerprint(path);
|
|
43
|
+
if (before === 'missing') {
|
|
44
|
+
throw new AssetSidecarRecoveryError(role === 'sidecar' ? 'sidecar_missing' : 'replacement_missing');
|
|
45
|
+
}
|
|
46
|
+
const value = readRegularBuffer(path, limit);
|
|
47
|
+
if (value === null) {
|
|
48
|
+
throw new AssetSidecarRecoveryError(role === 'sidecar' ? 'sidecar_missing' : 'replacement_missing');
|
|
49
|
+
}
|
|
50
|
+
const after = regularFileFingerprint(path);
|
|
51
|
+
if (before !== after) {
|
|
52
|
+
throw new AssetSidecarRecoveryError(role === 'sidecar' ? 'sidecar_changed' : 'replacement_changed');
|
|
53
|
+
}
|
|
54
|
+
return { bytes: value, fingerprint: after };
|
|
55
|
+
}
|
|
56
|
+
catch (error) {
|
|
57
|
+
if (error instanceof AssetStoreReadLimitError) {
|
|
58
|
+
throw new AssetSidecarRecoveryError(role === 'sidecar' ? 'sidecar_too_large' : 'replacement_too_large');
|
|
59
|
+
}
|
|
60
|
+
throw error;
|
|
61
|
+
}
|
|
62
|
+
}
|
|
63
|
+
function parseCurrent(bytes, sidecar) {
|
|
64
|
+
const hex = digest(bytes);
|
|
65
|
+
try {
|
|
66
|
+
const text = new TextDecoder('utf-8', { fatal: true }).decode(bytes);
|
|
67
|
+
return { bytes, text, parsed: parseRecoveryJsonl(text, RECORD_PARSERS[sidecar]), digest: `sha256:${hex}` };
|
|
68
|
+
}
|
|
69
|
+
catch {
|
|
70
|
+
return {
|
|
71
|
+
bytes,
|
|
72
|
+
text: null,
|
|
73
|
+
parsed: {
|
|
74
|
+
records: [],
|
|
75
|
+
rows: bytes.length > 0 ? 1 : 0,
|
|
76
|
+
validRows: 0,
|
|
77
|
+
corruptRows: bytes.length > 0 ? 1 : 0,
|
|
78
|
+
unterminated: bytes.length > 0 && bytes[bytes.length - 1] !== 0x0a,
|
|
79
|
+
},
|
|
80
|
+
digest: `sha256:${hex}`,
|
|
81
|
+
};
|
|
82
|
+
}
|
|
83
|
+
}
|
|
84
|
+
function parseReplacement(bytes, sidecar) {
|
|
85
|
+
let text;
|
|
86
|
+
try {
|
|
87
|
+
text = new TextDecoder('utf-8', { fatal: true }).decode(bytes);
|
|
88
|
+
}
|
|
89
|
+
catch {
|
|
90
|
+
throw new AssetSidecarRecoveryError('replacement_invalid_utf8');
|
|
91
|
+
}
|
|
92
|
+
const parsed = parseRecoveryJsonl(text, RECORD_PARSERS[sidecar]);
|
|
93
|
+
if (parsed.corruptRows > 0)
|
|
94
|
+
throw new AssetSidecarRecoveryError('replacement_invalid');
|
|
95
|
+
if (parsed.unterminated)
|
|
96
|
+
throw new AssetSidecarRecoveryError('replacement_unterminated');
|
|
97
|
+
return { bytes, text, parsed, digest: `sha256:${digest(bytes)}` };
|
|
98
|
+
}
|
|
99
|
+
function parseRecoveryJsonl(raw, parseRecord) {
|
|
100
|
+
const records = [];
|
|
101
|
+
let rows = 0;
|
|
102
|
+
let corruptRows = 0;
|
|
103
|
+
for (const line of raw.split('\n')) {
|
|
104
|
+
if (!line.trim())
|
|
105
|
+
continue;
|
|
106
|
+
rows += 1;
|
|
107
|
+
try {
|
|
108
|
+
const original = JSON.parse(line);
|
|
109
|
+
const normalized = parseRecord(original);
|
|
110
|
+
if (normalized)
|
|
111
|
+
records.push({ original, normalized });
|
|
112
|
+
else
|
|
113
|
+
corruptRows += 1;
|
|
114
|
+
}
|
|
115
|
+
catch {
|
|
116
|
+
corruptRows += 1;
|
|
117
|
+
}
|
|
118
|
+
}
|
|
119
|
+
return {
|
|
120
|
+
records,
|
|
121
|
+
rows,
|
|
122
|
+
validRows: records.length,
|
|
123
|
+
corruptRows,
|
|
124
|
+
unterminated: raw.length > 0 && !raw.endsWith('\n'),
|
|
125
|
+
};
|
|
126
|
+
}
|
|
127
|
+
function canonicalRecords(records) {
|
|
128
|
+
return records.map((record) => canonicalize(record.original));
|
|
129
|
+
}
|
|
130
|
+
function compareHistory(currentRecords, replacementRecords) {
|
|
131
|
+
const current = canonicalRecords(currentRecords);
|
|
132
|
+
const replacement = canonicalRecords(replacementRecords);
|
|
133
|
+
if (current.some((record, index) => replacement[index] !== record)) {
|
|
134
|
+
throw new AssetSidecarRecoveryError('valid_history_missing');
|
|
135
|
+
}
|
|
136
|
+
const extra = replacement.slice(current.length);
|
|
137
|
+
const existing = new Set(current);
|
|
138
|
+
return {
|
|
139
|
+
preservedValidRows: current.length,
|
|
140
|
+
addedRecords: extra.length,
|
|
141
|
+
correctiveRecords: extra.filter((record) => !existing.has(record)).length,
|
|
142
|
+
added: replacementRecords.slice(current.length),
|
|
143
|
+
};
|
|
144
|
+
}
|
|
145
|
+
function isSafeCorrectiveRecord(sidecar, value) {
|
|
146
|
+
if (sidecar === 'asset-sync')
|
|
147
|
+
return true;
|
|
148
|
+
if (!value.normalized || typeof value.normalized !== 'object' || Array.isArray(value.normalized))
|
|
149
|
+
return false;
|
|
150
|
+
const record = value.normalized;
|
|
151
|
+
return sidecar === 'provenance'
|
|
152
|
+
? record['trusted'] === false
|
|
153
|
+
: record['state'] === 'quarantined' || record['state'] === 'rejected';
|
|
154
|
+
}
|
|
155
|
+
function fileReport(file) {
|
|
156
|
+
return {
|
|
157
|
+
rows: file.parsed.rows,
|
|
158
|
+
validRows: file.parsed.validRows,
|
|
159
|
+
corruptRows: file.parsed.corruptRows,
|
|
160
|
+
unterminated: file.parsed.unterminated,
|
|
161
|
+
digest: file.digest,
|
|
162
|
+
};
|
|
163
|
+
}
|
|
164
|
+
function ensureExactBackup(baseDir, sidecar, current) {
|
|
165
|
+
const backupDir = join(baseDir, 'sidecar-backups');
|
|
166
|
+
assertAssetStoreDirectory(baseDir);
|
|
167
|
+
let backupDirCreated = false;
|
|
168
|
+
if (!existsSync(backupDir)) {
|
|
169
|
+
try {
|
|
170
|
+
mkdirSync(backupDir, { recursive: false, mode: 0o700 });
|
|
171
|
+
backupDirCreated = true;
|
|
172
|
+
}
|
|
173
|
+
catch (error) {
|
|
174
|
+
if (!isErrno(error, 'EEXIST'))
|
|
175
|
+
throw error;
|
|
176
|
+
}
|
|
177
|
+
}
|
|
178
|
+
assertAssetStoreDirectory(backupDir);
|
|
179
|
+
if (backupDirCreated)
|
|
180
|
+
fsyncDirectoryBestEffort(baseDir);
|
|
181
|
+
const hex = current.digest.slice('sha256:'.length);
|
|
182
|
+
const backupId = `${sidecar}-${hex}.jsonl`;
|
|
183
|
+
const backupPath = join(backupDir, backupId);
|
|
184
|
+
try {
|
|
185
|
+
createBufferDurableExclusive(backupPath, current.bytes);
|
|
186
|
+
}
|
|
187
|
+
catch (error) {
|
|
188
|
+
if (!isErrno(error, 'EEXIST'))
|
|
189
|
+
throw error;
|
|
190
|
+
let existing;
|
|
191
|
+
try {
|
|
192
|
+
existing = readRegularBuffer(backupPath, current.bytes.length);
|
|
193
|
+
}
|
|
194
|
+
catch (readError) {
|
|
195
|
+
if (readError instanceof AssetStoreReadLimitError) {
|
|
196
|
+
throw new AssetSidecarRecoveryError('backup_conflict');
|
|
197
|
+
}
|
|
198
|
+
throw readError;
|
|
199
|
+
}
|
|
200
|
+
if (!existing?.equals(current.bytes))
|
|
201
|
+
throw new AssetSidecarRecoveryError('backup_conflict');
|
|
202
|
+
}
|
|
203
|
+
return backupId;
|
|
204
|
+
}
|
|
205
|
+
export function recoverAssetSidecar(opts) {
|
|
206
|
+
assertAssetStoreDirectory(opts.baseDir);
|
|
207
|
+
const maxFileBytes = recoveryFileLimit(opts.maxFileBytes);
|
|
208
|
+
const targetPath = join(opts.baseDir, SIDECAR_FILES[opts.sidecar]);
|
|
209
|
+
const lockPath = join(opts.baseDir, '.assetstore.lock');
|
|
210
|
+
let committed;
|
|
211
|
+
try {
|
|
212
|
+
return withAssetStoreLock(lockPath, () => {
|
|
213
|
+
const currentRead = readBounded(targetPath, maxFileBytes, 'sidecar');
|
|
214
|
+
const replacementRead = readBounded(opts.replacementPath, maxFileBytes, 'replacement');
|
|
215
|
+
const current = parseCurrent(currentRead.bytes, opts.sidecar);
|
|
216
|
+
const replacement = parseReplacement(replacementRead.bytes, opts.sidecar);
|
|
217
|
+
const history = compareHistory(current.parsed.records, replacement.parsed.records);
|
|
218
|
+
const degraded = current.parsed.corruptRows > 0 || current.parsed.unterminated;
|
|
219
|
+
const sameBytes = current.bytes.equals(replacement.bytes);
|
|
220
|
+
if (!degraded && !sameBytes)
|
|
221
|
+
throw new AssetSidecarRecoveryError('sidecar_not_degraded');
|
|
222
|
+
if (current.parsed.corruptRows > 0
|
|
223
|
+
&& opts.sidecar !== 'asset-sync'
|
|
224
|
+
&& history.correctiveRecords === 0) {
|
|
225
|
+
throw new AssetSidecarRecoveryError('corrective_record_required');
|
|
226
|
+
}
|
|
227
|
+
if (history.added.some((record) => !isSafeCorrectiveRecord(opts.sidecar, record))) {
|
|
228
|
+
throw new AssetSidecarRecoveryError('unsafe_corrective_record');
|
|
229
|
+
}
|
|
230
|
+
const acknowledgementRequired = degraded && !sameBytes;
|
|
231
|
+
const mode = opts.write ? 'write' : 'preview';
|
|
232
|
+
const baseReport = {
|
|
233
|
+
mode,
|
|
234
|
+
sidecar: opts.sidecar,
|
|
235
|
+
current: fileReport(current),
|
|
236
|
+
replacement: {
|
|
237
|
+
...fileReport(replacement),
|
|
238
|
+
preservedValidRows: history.preservedValidRows,
|
|
239
|
+
addedRecords: history.addedRecords,
|
|
240
|
+
correctiveRecords: history.correctiveRecords,
|
|
241
|
+
},
|
|
242
|
+
};
|
|
243
|
+
if (sameBytes) {
|
|
244
|
+
return {
|
|
245
|
+
...baseReport,
|
|
246
|
+
changed: false,
|
|
247
|
+
wouldWrite: false,
|
|
248
|
+
acknowledgementRequired: false,
|
|
249
|
+
};
|
|
250
|
+
}
|
|
251
|
+
if (!opts.write) {
|
|
252
|
+
return {
|
|
253
|
+
...baseReport,
|
|
254
|
+
changed: false,
|
|
255
|
+
wouldWrite: true,
|
|
256
|
+
acknowledgementRequired,
|
|
257
|
+
};
|
|
258
|
+
}
|
|
259
|
+
if (!opts.acknowledgeCorruptHistory) {
|
|
260
|
+
throw new AssetSidecarRecoveryError('acknowledgement_required');
|
|
261
|
+
}
|
|
262
|
+
opts.deps?.beforeBackup?.();
|
|
263
|
+
const backupId = ensureExactBackup(opts.baseDir, opts.sidecar, current);
|
|
264
|
+
opts.deps?.beforeReplace?.();
|
|
265
|
+
if (regularFileFingerprint(targetPath) !== currentRead.fingerprint) {
|
|
266
|
+
throw new AssetSidecarRecoveryError('sidecar_changed');
|
|
267
|
+
}
|
|
268
|
+
if (regularFileFingerprint(opts.replacementPath) !== replacementRead.fingerprint) {
|
|
269
|
+
throw new AssetSidecarRecoveryError('replacement_changed');
|
|
270
|
+
}
|
|
271
|
+
replaceUtf8Durable(targetPath, replacement.text);
|
|
272
|
+
committed = {
|
|
273
|
+
...baseReport,
|
|
274
|
+
changed: true,
|
|
275
|
+
wouldWrite: false,
|
|
276
|
+
acknowledgementRequired: false,
|
|
277
|
+
backupId,
|
|
278
|
+
};
|
|
279
|
+
return committed;
|
|
280
|
+
}, opts.deps?.lock);
|
|
281
|
+
}
|
|
282
|
+
catch (error) {
|
|
283
|
+
if (committed && error instanceof LockReleaseError) {
|
|
284
|
+
return { ...committed, lockReleaseWarning: error.reason };
|
|
285
|
+
}
|
|
286
|
+
throw error;
|
|
287
|
+
}
|
|
288
|
+
}
|
|
@@ -0,0 +1,81 @@
|
|
|
1
|
+
import { acquireLock, releaseLock } from '../util/fileLock.js';
|
|
2
|
+
import { type UnsafeAssetStorePathReason } from './assetStoreStorage.js';
|
|
3
|
+
import type { AssetKind } from './provider.js';
|
|
4
|
+
import { type AssetSidecarKind } from './assetSidecarRecords.js';
|
|
5
|
+
export type AssetStoreHealthStatus = 'healthy' | 'degraded' | 'unsafe' | 'unavailable';
|
|
6
|
+
export type AssetStoreFileStatus = 'missing' | 'ok' | 'degraded' | 'unsafe' | 'unavailable';
|
|
7
|
+
export interface AssetStoreFileHealth {
|
|
8
|
+
kind: AssetKind;
|
|
9
|
+
file: string;
|
|
10
|
+
status: AssetStoreFileStatus;
|
|
11
|
+
bytes: number;
|
|
12
|
+
rows: number;
|
|
13
|
+
validRows: number;
|
|
14
|
+
uniqueAssets: number;
|
|
15
|
+
duplicateRows: number;
|
|
16
|
+
corruptRows: number;
|
|
17
|
+
hashMismatchRows: number;
|
|
18
|
+
/**
|
|
19
|
+
* Rows that fail content-hash verification but carry an active, content-bound unverified provenance waiver.
|
|
20
|
+
* They remain untrusted and are counted separately instead of being misclassified as store corruption.
|
|
21
|
+
*/
|
|
22
|
+
unverifiedRows: number;
|
|
23
|
+
schemaInvalidRows: number;
|
|
24
|
+
unterminated: boolean;
|
|
25
|
+
reason?: UnsafeAssetStorePathReason | 'base_directory' | 'lock_unavailable' | 'read_unavailable' | 'scan_limit_exceeded';
|
|
26
|
+
}
|
|
27
|
+
export interface AssetStoreHealthTotals {
|
|
28
|
+
files: number;
|
|
29
|
+
missingFiles: number;
|
|
30
|
+
unsafeFiles: number;
|
|
31
|
+
unavailableFiles: number;
|
|
32
|
+
bytes: number;
|
|
33
|
+
rows: number;
|
|
34
|
+
validRows: number;
|
|
35
|
+
uniqueAssets: number;
|
|
36
|
+
duplicateRows: number;
|
|
37
|
+
corruptRows: number;
|
|
38
|
+
hashMismatchRows: number;
|
|
39
|
+
unverifiedRows: number;
|
|
40
|
+
schemaInvalidRows: number;
|
|
41
|
+
unterminatedFiles: number;
|
|
42
|
+
}
|
|
43
|
+
export interface AssetStoreSidecarHealth {
|
|
44
|
+
kind: AssetSidecarKind;
|
|
45
|
+
file: string;
|
|
46
|
+
status: AssetStoreFileStatus;
|
|
47
|
+
bytes: number;
|
|
48
|
+
rows: number;
|
|
49
|
+
validRows: number;
|
|
50
|
+
corruptRows: number;
|
|
51
|
+
unterminated: boolean;
|
|
52
|
+
reason?: AssetStoreFileHealth['reason'];
|
|
53
|
+
}
|
|
54
|
+
export interface AssetStoreSidecarHealthTotals {
|
|
55
|
+
files: number;
|
|
56
|
+
missingFiles: number;
|
|
57
|
+
unsafeFiles: number;
|
|
58
|
+
unavailableFiles: number;
|
|
59
|
+
bytes: number;
|
|
60
|
+
rows: number;
|
|
61
|
+
validRows: number;
|
|
62
|
+
corruptRows: number;
|
|
63
|
+
unterminatedFiles: number;
|
|
64
|
+
}
|
|
65
|
+
export interface AssetStoreHealthReport {
|
|
66
|
+
ok: boolean;
|
|
67
|
+
status: AssetStoreHealthStatus;
|
|
68
|
+
totals: AssetStoreHealthTotals;
|
|
69
|
+
files: AssetStoreFileHealth[];
|
|
70
|
+
sidecarTotals: AssetStoreSidecarHealthTotals;
|
|
71
|
+
sidecars: AssetStoreSidecarHealth[];
|
|
72
|
+
}
|
|
73
|
+
export interface InspectLocalAssetStoreOptions {
|
|
74
|
+
maxFileBytes?: number;
|
|
75
|
+
}
|
|
76
|
+
export interface InspectLocalAssetStoreDeps {
|
|
77
|
+
acquireLock?: typeof acquireLock;
|
|
78
|
+
releaseLock?: typeof releaseLock;
|
|
79
|
+
}
|
|
80
|
+
export declare const DEFAULT_ASSET_HEALTH_MAX_FILE_BYTES: number;
|
|
81
|
+
export declare function inspectLocalAssetStore(baseDir: string, opts?: InspectLocalAssetStoreOptions, deps?: InspectLocalAssetStoreDeps): AssetStoreHealthReport;
|
|
@@ -0,0 +1,319 @@
|
|
|
1
|
+
import { lstatSync } from 'node:fs';
|
|
2
|
+
import { join } from 'node:path';
|
|
3
|
+
import { acquireLock, releaseLock } from '../util/fileLock.js';
|
|
4
|
+
import { computeAssetId, validateWire, verifyAssetId } from '../wire/index.js';
|
|
5
|
+
import { LOCAL_ASSET_FILES } from './assetStoreLayout.js';
|
|
6
|
+
import { assertOptionalRegularFile, isReliableAssetStoreLockRelease, readUtf8Regular, UnsafeAssetStorePathError, } from './assetStoreStorage.js';
|
|
7
|
+
import { isActiveUnverifiedProvenance } from './provenance.js';
|
|
8
|
+
import { parseAssetSyncSidecarRecord, parseProvenanceRecord, parseReviewRecord, parseSidecarJsonl, } from './assetSidecarRecords.js';
|
|
9
|
+
export const DEFAULT_ASSET_HEALTH_MAX_FILE_BYTES = 64 * 1024 * 1024;
|
|
10
|
+
const LOCAL_ASSET_SIDECARS = [
|
|
11
|
+
{ kind: 'provenance', file: 'provenance.jsonl', parseRecord: parseProvenanceRecord },
|
|
12
|
+
{ kind: 'review', file: 'review.jsonl', parseRecord: parseReviewRecord },
|
|
13
|
+
{ kind: 'asset-sync', file: 'asset-sync.jsonl', parseRecord: parseAssetSyncSidecarRecord },
|
|
14
|
+
];
|
|
15
|
+
function healthScanLimit(value) {
|
|
16
|
+
if (value === undefined || !Number.isFinite(value))
|
|
17
|
+
return DEFAULT_ASSET_HEALTH_MAX_FILE_BYTES;
|
|
18
|
+
return Math.min(DEFAULT_ASSET_HEALTH_MAX_FILE_BYTES, Math.max(1, Math.floor(value)));
|
|
19
|
+
}
|
|
20
|
+
function isErrno(error, code) {
|
|
21
|
+
return typeof error === 'object' && error !== null && error.code === code;
|
|
22
|
+
}
|
|
23
|
+
function emptyFile(kind, file, status, reason) {
|
|
24
|
+
return {
|
|
25
|
+
kind,
|
|
26
|
+
file,
|
|
27
|
+
status,
|
|
28
|
+
bytes: 0,
|
|
29
|
+
rows: 0,
|
|
30
|
+
validRows: 0,
|
|
31
|
+
uniqueAssets: 0,
|
|
32
|
+
duplicateRows: 0,
|
|
33
|
+
corruptRows: 0,
|
|
34
|
+
hashMismatchRows: 0,
|
|
35
|
+
unverifiedRows: 0,
|
|
36
|
+
schemaInvalidRows: 0,
|
|
37
|
+
unterminated: false,
|
|
38
|
+
...(reason ? { reason } : {}),
|
|
39
|
+
};
|
|
40
|
+
}
|
|
41
|
+
function allFiles(status, reason) {
|
|
42
|
+
return Object.entries(LOCAL_ASSET_FILES)
|
|
43
|
+
.map(([kind, file]) => emptyFile(kind, file, status, reason));
|
|
44
|
+
}
|
|
45
|
+
function emptySidecar(kind, file, status, reason) {
|
|
46
|
+
return {
|
|
47
|
+
kind,
|
|
48
|
+
file,
|
|
49
|
+
status,
|
|
50
|
+
bytes: 0,
|
|
51
|
+
rows: 0,
|
|
52
|
+
validRows: 0,
|
|
53
|
+
corruptRows: 0,
|
|
54
|
+
unterminated: false,
|
|
55
|
+
...(reason ? { reason } : {}),
|
|
56
|
+
};
|
|
57
|
+
}
|
|
58
|
+
function allSidecars(status, reason) {
|
|
59
|
+
return LOCAL_ASSET_SIDECARS.map(({ kind, file }) => emptySidecar(kind, file, status, reason));
|
|
60
|
+
}
|
|
61
|
+
function summarize(files, sidecars) {
|
|
62
|
+
const totals = {
|
|
63
|
+
files: files.length,
|
|
64
|
+
missingFiles: files.filter((file) => file.status === 'missing').length,
|
|
65
|
+
unsafeFiles: files.filter((file) => file.status === 'unsafe').length,
|
|
66
|
+
unavailableFiles: files.filter((file) => file.status === 'unavailable').length,
|
|
67
|
+
bytes: files.reduce((sum, file) => sum + file.bytes, 0),
|
|
68
|
+
rows: files.reduce((sum, file) => sum + file.rows, 0),
|
|
69
|
+
validRows: files.reduce((sum, file) => sum + file.validRows, 0),
|
|
70
|
+
uniqueAssets: files.reduce((sum, file) => sum + file.uniqueAssets, 0),
|
|
71
|
+
duplicateRows: files.reduce((sum, file) => sum + file.duplicateRows, 0),
|
|
72
|
+
corruptRows: files.reduce((sum, file) => sum + file.corruptRows, 0),
|
|
73
|
+
hashMismatchRows: files.reduce((sum, file) => sum + file.hashMismatchRows, 0),
|
|
74
|
+
unverifiedRows: files.reduce((sum, file) => sum + file.unverifiedRows, 0),
|
|
75
|
+
schemaInvalidRows: files.reduce((sum, file) => sum + file.schemaInvalidRows, 0),
|
|
76
|
+
unterminatedFiles: files.filter((file) => file.unterminated).length,
|
|
77
|
+
};
|
|
78
|
+
const sidecarTotals = {
|
|
79
|
+
files: sidecars.length,
|
|
80
|
+
missingFiles: sidecars.filter((file) => file.status === 'missing').length,
|
|
81
|
+
unsafeFiles: sidecars.filter((file) => file.status === 'unsafe').length,
|
|
82
|
+
unavailableFiles: sidecars.filter((file) => file.status === 'unavailable').length,
|
|
83
|
+
bytes: sidecars.reduce((sum, file) => sum + file.bytes, 0),
|
|
84
|
+
rows: sidecars.reduce((sum, file) => sum + file.rows, 0),
|
|
85
|
+
validRows: sidecars.reduce((sum, file) => sum + file.validRows, 0),
|
|
86
|
+
corruptRows: sidecars.reduce((sum, file) => sum + file.corruptRows, 0),
|
|
87
|
+
unterminatedFiles: sidecars.filter((file) => file.unterminated).length,
|
|
88
|
+
};
|
|
89
|
+
const status = totals.unsafeFiles > 0 || sidecarTotals.unsafeFiles > 0
|
|
90
|
+
? 'unsafe'
|
|
91
|
+
: totals.unavailableFiles > 0 || sidecarTotals.unavailableFiles > 0
|
|
92
|
+
? 'unavailable'
|
|
93
|
+
: totals.duplicateRows > 0
|
|
94
|
+
|| totals.corruptRows > 0
|
|
95
|
+
|| totals.hashMismatchRows > 0
|
|
96
|
+
|| totals.schemaInvalidRows > 0
|
|
97
|
+
|| totals.unterminatedFiles > 0
|
|
98
|
+
|| sidecarTotals.corruptRows > 0
|
|
99
|
+
|| sidecarTotals.unterminatedFiles > 0
|
|
100
|
+
? 'degraded'
|
|
101
|
+
: 'healthy';
|
|
102
|
+
return {
|
|
103
|
+
ok: status === 'healthy',
|
|
104
|
+
status,
|
|
105
|
+
totals,
|
|
106
|
+
files: [...files],
|
|
107
|
+
sidecarTotals,
|
|
108
|
+
sidecars: [...sidecars],
|
|
109
|
+
};
|
|
110
|
+
}
|
|
111
|
+
function inspectFile(baseDir, kind, file, maxFileBytes, unverifiedContentIds) {
|
|
112
|
+
try {
|
|
113
|
+
const path = join(baseDir, file);
|
|
114
|
+
const stat = assertOptionalRegularFile(path);
|
|
115
|
+
if (stat === null)
|
|
116
|
+
return emptyFile(kind, file, 'missing');
|
|
117
|
+
if (stat.size > maxFileBytes) {
|
|
118
|
+
return { ...emptyFile(kind, file, 'unavailable', 'scan_limit_exceeded'), bytes: stat.size };
|
|
119
|
+
}
|
|
120
|
+
const raw = readUtf8Regular(path, maxFileBytes);
|
|
121
|
+
if (raw === null)
|
|
122
|
+
return emptyFile(kind, file, 'unavailable', 'read_unavailable');
|
|
123
|
+
const rows = raw.split('\n').filter((line) => line.trim().length > 0);
|
|
124
|
+
const seen = new Set();
|
|
125
|
+
let validRows = 0;
|
|
126
|
+
let duplicateRows = 0;
|
|
127
|
+
let corruptRows = 0;
|
|
128
|
+
let hashMismatchRows = 0;
|
|
129
|
+
let unverifiedRows = 0;
|
|
130
|
+
let schemaInvalidRows = 0;
|
|
131
|
+
for (const line of rows) {
|
|
132
|
+
try {
|
|
133
|
+
const value = JSON.parse(line);
|
|
134
|
+
if (!value || typeof value !== 'object' || Array.isArray(value)) {
|
|
135
|
+
corruptRows += 1;
|
|
136
|
+
continue;
|
|
137
|
+
}
|
|
138
|
+
const record = value;
|
|
139
|
+
const assetId = typeof record['asset_id'] === 'string' ? record['asset_id'].trim() : '';
|
|
140
|
+
if (!assetId || record['type'] !== kind) {
|
|
141
|
+
corruptRows += 1;
|
|
142
|
+
continue;
|
|
143
|
+
}
|
|
144
|
+
if (seen.has(assetId))
|
|
145
|
+
duplicateRows += 1;
|
|
146
|
+
else
|
|
147
|
+
seen.add(assetId);
|
|
148
|
+
const hashMatches = verifyAssetId(record);
|
|
149
|
+
if (!hashMatches) {
|
|
150
|
+
// A hash mismatch is corruption UNLESS provenance says this id is an unverified hub reuse: the hub
|
|
151
|
+
// rewrote the delivered bytes and reuse froze them untrusted on purpose (#570). That is expected, not
|
|
152
|
+
// store rot, so it lands in the benign unverifiedRows bucket and never degrades the store.
|
|
153
|
+
if (unverifiedContentIds.get(assetId) === computeAssetId(record))
|
|
154
|
+
unverifiedRows += 1;
|
|
155
|
+
else
|
|
156
|
+
hashMismatchRows += 1;
|
|
157
|
+
}
|
|
158
|
+
const schemaValid = kind === 'AntiGene' || validateWire(record).ok;
|
|
159
|
+
if (!schemaValid) {
|
|
160
|
+
schemaInvalidRows += 1;
|
|
161
|
+
}
|
|
162
|
+
if (hashMatches && schemaValid)
|
|
163
|
+
validRows += 1;
|
|
164
|
+
}
|
|
165
|
+
catch {
|
|
166
|
+
corruptRows += 1;
|
|
167
|
+
}
|
|
168
|
+
}
|
|
169
|
+
const unterminated = raw.length > 0 && !raw.endsWith('\n');
|
|
170
|
+
const status = duplicateRows > 0
|
|
171
|
+
|| corruptRows > 0
|
|
172
|
+
|| hashMismatchRows > 0
|
|
173
|
+
|| schemaInvalidRows > 0
|
|
174
|
+
|| unterminated
|
|
175
|
+
? 'degraded'
|
|
176
|
+
: 'ok';
|
|
177
|
+
return {
|
|
178
|
+
kind,
|
|
179
|
+
file,
|
|
180
|
+
status,
|
|
181
|
+
bytes: Buffer.byteLength(raw, 'utf8'),
|
|
182
|
+
rows: rows.length,
|
|
183
|
+
validRows,
|
|
184
|
+
uniqueAssets: seen.size,
|
|
185
|
+
duplicateRows,
|
|
186
|
+
corruptRows,
|
|
187
|
+
hashMismatchRows,
|
|
188
|
+
unverifiedRows,
|
|
189
|
+
schemaInvalidRows,
|
|
190
|
+
unterminated,
|
|
191
|
+
};
|
|
192
|
+
}
|
|
193
|
+
catch (error) {
|
|
194
|
+
if (error instanceof UnsafeAssetStorePathError)
|
|
195
|
+
return emptyFile(kind, file, 'unsafe', error.reason);
|
|
196
|
+
return emptyFile(kind, file, 'unavailable', 'read_unavailable');
|
|
197
|
+
}
|
|
198
|
+
}
|
|
199
|
+
/**
|
|
200
|
+
* Asset ids whose latest provenance decision marks a supported unverified frozen ingest.
|
|
201
|
+
* Read directly from `<baseDir>/provenance.jsonl` — NOT via ProvenanceStore — because inspectLocalAssetStore
|
|
202
|
+
* already holds the shared `.assetstore.lock`, and ProvenanceStore would try to re-acquire it. A missing or
|
|
203
|
+
* unreadable sidecar yields an empty map: without provenance every mismatch stays classified as corruption.
|
|
204
|
+
*/
|
|
205
|
+
function readUnverifiedReuseContentIds(baseDir, maxFileBytes) {
|
|
206
|
+
let raw;
|
|
207
|
+
try {
|
|
208
|
+
raw = readUtf8Regular(join(baseDir, 'provenance.jsonl'), maxFileBytes);
|
|
209
|
+
}
|
|
210
|
+
catch {
|
|
211
|
+
return new Map();
|
|
212
|
+
}
|
|
213
|
+
if (raw === null)
|
|
214
|
+
return new Map();
|
|
215
|
+
const ids = new Map();
|
|
216
|
+
for (const record of parseSidecarJsonl(raw, parseProvenanceRecord).records) {
|
|
217
|
+
const frozenContentId = record.frozenContentId;
|
|
218
|
+
if (frozenContentId && isActiveUnverifiedProvenance(record, frozenContentId)) {
|
|
219
|
+
ids.set(record.assetId, frozenContentId);
|
|
220
|
+
}
|
|
221
|
+
else
|
|
222
|
+
ids.delete(record.assetId);
|
|
223
|
+
}
|
|
224
|
+
return ids;
|
|
225
|
+
}
|
|
226
|
+
function inspectSidecar(baseDir, definition, maxFileBytes) {
|
|
227
|
+
const { kind, file, parseRecord } = definition;
|
|
228
|
+
try {
|
|
229
|
+
const path = join(baseDir, file);
|
|
230
|
+
const stat = assertOptionalRegularFile(path);
|
|
231
|
+
if (stat === null)
|
|
232
|
+
return emptySidecar(kind, file, 'missing');
|
|
233
|
+
if (stat.size > maxFileBytes) {
|
|
234
|
+
return { ...emptySidecar(kind, file, 'unavailable', 'scan_limit_exceeded'), bytes: stat.size };
|
|
235
|
+
}
|
|
236
|
+
const raw = readUtf8Regular(path, maxFileBytes);
|
|
237
|
+
if (raw === null)
|
|
238
|
+
return emptySidecar(kind, file, 'unavailable', 'read_unavailable');
|
|
239
|
+
const parsed = parseSidecarJsonl(raw, parseRecord);
|
|
240
|
+
const status = parsed.corruptRows > 0 || parsed.unterminated ? 'degraded' : 'ok';
|
|
241
|
+
return {
|
|
242
|
+
kind,
|
|
243
|
+
file,
|
|
244
|
+
status,
|
|
245
|
+
bytes: Buffer.byteLength(raw, 'utf8'),
|
|
246
|
+
rows: parsed.rows,
|
|
247
|
+
validRows: parsed.validRows,
|
|
248
|
+
corruptRows: parsed.corruptRows,
|
|
249
|
+
unterminated: parsed.unterminated,
|
|
250
|
+
};
|
|
251
|
+
}
|
|
252
|
+
catch (error) {
|
|
253
|
+
if (error instanceof UnsafeAssetStorePathError)
|
|
254
|
+
return emptySidecar(kind, file, 'unsafe', error.reason);
|
|
255
|
+
return emptySidecar(kind, file, 'unavailable', 'read_unavailable');
|
|
256
|
+
}
|
|
257
|
+
}
|
|
258
|
+
function directoryState(baseDir) {
|
|
259
|
+
try {
|
|
260
|
+
const stat = lstatSync(baseDir);
|
|
261
|
+
if (stat.isSymbolicLink() || !stat.isDirectory())
|
|
262
|
+
return 'unsafe';
|
|
263
|
+
return 'safe';
|
|
264
|
+
}
|
|
265
|
+
catch (error) {
|
|
266
|
+
if (isErrno(error, 'ENOENT'))
|
|
267
|
+
return 'missing';
|
|
268
|
+
return 'unavailable';
|
|
269
|
+
}
|
|
270
|
+
}
|
|
271
|
+
export function inspectLocalAssetStore(baseDir, opts = {}, deps = {}) {
|
|
272
|
+
const state = directoryState(baseDir);
|
|
273
|
+
if (state === 'missing')
|
|
274
|
+
return summarize(allFiles('missing'), allSidecars('missing'));
|
|
275
|
+
if (state === 'unsafe') {
|
|
276
|
+
return summarize(allFiles('unsafe', 'base_directory'), allSidecars('unsafe', 'base_directory'));
|
|
277
|
+
}
|
|
278
|
+
if (state === 'unavailable') {
|
|
279
|
+
return summarize(allFiles('unavailable', 'read_unavailable'), allSidecars('unavailable', 'read_unavailable'));
|
|
280
|
+
}
|
|
281
|
+
const lockPath = join(baseDir, '.assetstore.lock');
|
|
282
|
+
try {
|
|
283
|
+
assertOptionalRegularFile(lockPath, 'lock_file');
|
|
284
|
+
}
|
|
285
|
+
catch (error) {
|
|
286
|
+
if (error instanceof UnsafeAssetStorePathError) {
|
|
287
|
+
return summarize(allFiles('unsafe', error.reason), allSidecars('unsafe', error.reason));
|
|
288
|
+
}
|
|
289
|
+
return summarize(allFiles('unavailable', 'lock_unavailable'), allSidecars('unavailable', 'lock_unavailable'));
|
|
290
|
+
}
|
|
291
|
+
try {
|
|
292
|
+
(deps.acquireLock ?? acquireLock)(lockPath);
|
|
293
|
+
}
|
|
294
|
+
catch {
|
|
295
|
+
return summarize(allFiles('unavailable', 'lock_unavailable'), allSidecars('unavailable', 'lock_unavailable'));
|
|
296
|
+
}
|
|
297
|
+
let report;
|
|
298
|
+
try {
|
|
299
|
+
const maxFileBytes = healthScanLimit(opts.maxFileBytes);
|
|
300
|
+
// Read the unverified-reuse set once, under the lock we already hold, so every asset file classifies its
|
|
301
|
+
// hash mismatches consistently against the same bounded provenance snapshot.
|
|
302
|
+
const unverifiedContentIds = readUnverifiedReuseContentIds(baseDir, maxFileBytes);
|
|
303
|
+
report = summarize(Object.entries(LOCAL_ASSET_FILES)
|
|
304
|
+
.map(([kind, file]) => inspectFile(baseDir, kind, file, maxFileBytes, unverifiedContentIds)), LOCAL_ASSET_SIDECARS.map((definition) => inspectSidecar(baseDir, definition, maxFileBytes)));
|
|
305
|
+
}
|
|
306
|
+
catch {
|
|
307
|
+
report = summarize(allFiles('unavailable', 'read_unavailable'), allSidecars('unavailable', 'read_unavailable'));
|
|
308
|
+
}
|
|
309
|
+
try {
|
|
310
|
+
const released = (deps.releaseLock ?? releaseLock)(lockPath);
|
|
311
|
+
if (!isReliableAssetStoreLockRelease(released)) {
|
|
312
|
+
return summarize(allFiles('unavailable', 'lock_unavailable'), allSidecars('unavailable', 'lock_unavailable'));
|
|
313
|
+
}
|
|
314
|
+
}
|
|
315
|
+
catch {
|
|
316
|
+
return summarize(allFiles('unavailable', 'lock_unavailable'), allSidecars('unavailable', 'lock_unavailable'));
|
|
317
|
+
}
|
|
318
|
+
return report;
|
|
319
|
+
}
|