neurain 0.1.0-alpha.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +19 -0
- package/LICENSE +57 -0
- package/README.md +205 -0
- package/SECURITY.md +22 -0
- package/bin/neurain.mjs +7 -0
- package/docs/comparison-mem0.en.md +22 -0
- package/docs/connect-claude.en.md +48 -0
- package/docs/connect-claude.kr.md +51 -0
- package/docs/connect-codex.en.md +38 -0
- package/docs/connect-codex.kr.md +40 -0
- package/docs/connect-gemini.en.md +71 -0
- package/docs/connect-gemini.kr.md +71 -0
- package/docs/connect-runtime.en.md +61 -0
- package/docs/connect-runtime.kr.md +61 -0
- package/docs/development-status.en.md +157 -0
- package/docs/development-status.kr.md +157 -0
- package/docs/knowledge-os.en.md +105 -0
- package/docs/knowledge-os.kr.md +106 -0
- package/docs/pricing.en.md +14 -0
- package/docs/privacy-and-data-flow.en.md +25 -0
- package/docs/public-saas-readiness.en.md +39 -0
- package/docs/quickstart.en.md +64 -0
- package/docs/quickstart.kr.md +64 -0
- package/docs/release-checklist.en.md +38 -0
- package/docs/safety.en.md +36 -0
- package/docs/self-improvement-90-roadmap.en.md +429 -0
- package/docs/self-improvement-90-roadmap.kr.md +429 -0
- package/docs/self-improving-workflows.en.md +163 -0
- package/docs/self-improving-workflows.kr.md +163 -0
- package/docs/support.en.md +17 -0
- package/docs/troubleshooting.en.md +35 -0
- package/package.json +36 -0
- package/src/cli.mjs +261 -0
- package/src/core/adopt.mjs +304 -0
- package/src/core/answer_eval.mjs +450 -0
- package/src/core/capabilities.mjs +217 -0
- package/src/core/capture_durable.mjs +181 -0
- package/src/core/classify.mjs +237 -0
- package/src/core/compile_desk.mjs +324 -0
- package/src/core/complete.mjs +108 -0
- package/src/core/config.mjs +142 -0
- package/src/core/connect.mjs +355 -0
- package/src/core/curator.mjs +351 -0
- package/src/core/daemon.mjs +536 -0
- package/src/core/digest.mjs +155 -0
- package/src/core/doctor.mjs +115 -0
- package/src/core/durable.mjs +96 -0
- package/src/core/envelope.mjs +97 -0
- package/src/core/flush.mjs +190 -0
- package/src/core/fs.mjs +121 -0
- package/src/core/init.mjs +194 -0
- package/src/core/journal.mjs +269 -0
- package/src/core/labels.mjs +117 -0
- package/src/core/lessons.mjs +793 -0
- package/src/core/lifecycle.mjs +1138 -0
- package/src/core/link_check.mjs +180 -0
- package/src/core/live_cases.mjs +221 -0
- package/src/core/onboard.mjs +175 -0
- package/src/core/plan_receipt.mjs +177 -0
- package/src/core/plan_writeback.mjs +176 -0
- package/src/core/queue.mjs +62 -0
- package/src/core/queue_archive.mjs +87 -0
- package/src/core/queue_model.mjs +161 -0
- package/src/core/queue_write.mjs +28 -0
- package/src/core/recall.mjs +1802 -0
- package/src/core/recall_bench.mjs +275 -0
- package/src/core/recall_corpus.mjs +152 -0
- package/src/core/recall_facts.mjs +233 -0
- package/src/core/recall_intel.mjs +233 -0
- package/src/core/recall_lexical.mjs +269 -0
- package/src/core/recap.mjs +78 -0
- package/src/core/review_queue.mjs +131 -0
- package/src/core/review_worker.mjs +284 -0
- package/src/core/route.mjs +73 -0
- package/src/core/safety.mjs +57 -0
- package/src/core/scheduler.mjs +697 -0
- package/src/core/search.mjs +54 -0
- package/src/core/secret_scan.mjs +143 -0
- package/src/core/semantic.mjs +187 -0
- package/src/core/source_digest.mjs +56 -0
- package/src/core/source_digest_gen.mjs +311 -0
- package/src/core/stage.mjs +105 -0
- package/src/core/status.mjs +175 -0
- package/src/core/vault_state.mjs +115 -0
- package/src/core/watch.mjs +282 -0
- package/src/core/wiki_log.mjs +29 -0
- package/src/core/wrap.mjs +62 -0
- package/src/mcp/server.mjs +865 -0
- package/templates/starter-vault/README.md +9 -0
|
@@ -0,0 +1,311 @@
|
|
|
1
|
+
// `source-digest` command (W-B). The manifest GENERATOR: scans the raw source
|
|
2
|
+
// tree, fingerprints each file (reusing the prior sha when size+mtime are
|
|
3
|
+
// unchanged), infers compile status, and scores high-value sources for deep
|
|
4
|
+
// compile. Its only write is the source-digest manifest the engine read side
|
|
5
|
+
// (source_digest.mjs) already consumes; dry by default, writes only with --write.
|
|
6
|
+
// Faithful port of the vault neurain-source-digest tool, EXCEPT the path
|
|
7
|
+
// sensitivity fallback, which uses the W-A label resolver + registry baseline
|
|
8
|
+
// instead of the vault's hardcoded area-name regex (no vault identifiers leak
|
|
9
|
+
// into the engine; on the reference vault the result is equivalent).
|
|
10
|
+
import crypto from 'node:crypto';
|
|
11
|
+
import fs from 'node:fs';
|
|
12
|
+
import path from 'node:path';
|
|
13
|
+
import { absPath, ensureDir, readText, relPath, timestamp, walkFiles } from './fs.mjs';
|
|
14
|
+
import { recallConfig, vaultConfig } from './config.mjs';
|
|
15
|
+
import { createSensitivityResolver } from './labels.mjs';
|
|
16
|
+
import { loadAreaIntel } from './classify.mjs';
|
|
17
|
+
import { atomicWriteJson } from './durable.mjs';
|
|
18
|
+
import { readJsonSafe } from './vault_state.mjs';
|
|
19
|
+
|
|
20
|
+
const TERMINAL_STATUSES = new Set(['compiled', 'flushed', 'archived', 'superseded']);
|
|
21
|
+
const HIGH_VALUE_EXTS = new Set(['.pdf', '.docx', '.xlsx', '.xls', '.pptx', '.csv']);
|
|
22
|
+
|
|
23
|
+
export async function sourceDigestCommand(args) {
|
|
24
|
+
const root = absPath(args._[0] || args.root || process.cwd());
|
|
25
|
+
const vaultCfg = vaultConfig(root);
|
|
26
|
+
const recallCfg = recallConfig(root);
|
|
27
|
+
const intel = loadAreaIntel(root, vaultCfg);
|
|
28
|
+
const resolver = createSensitivityResolver(root, recallCfg);
|
|
29
|
+
|
|
30
|
+
const scope = String(args.scope || args.path || vaultCfg.raw_dir);
|
|
31
|
+
const manifestRel = String(args.manifest || vaultCfg.source_digest_manifest);
|
|
32
|
+
const write = Boolean(args.write || args.update);
|
|
33
|
+
const rehash = Boolean(args.rehash || args.full);
|
|
34
|
+
const top = Number(args.top || 20);
|
|
35
|
+
const now = timestamp();
|
|
36
|
+
const manifestAbs = path.join(root, manifestRel);
|
|
37
|
+
const previous = readJsonSafe(manifestAbs, null) || { version: 1, updated_at: '', files: {} };
|
|
38
|
+
|
|
39
|
+
const files = scanSourceFiles(root, scope, vaultCfg);
|
|
40
|
+
const nextFiles = {};
|
|
41
|
+
const newItems = [];
|
|
42
|
+
const changedItems = [];
|
|
43
|
+
const unchangedItems = [];
|
|
44
|
+
const highValueCandidates = [];
|
|
45
|
+
let reusedHashCount = 0;
|
|
46
|
+
|
|
47
|
+
for (const abs of files) {
|
|
48
|
+
const rel = relPath(root, abs);
|
|
49
|
+
const stat = fs.statSync(abs);
|
|
50
|
+
const previousEntry = (previous.files && previous.files[rel]) || null;
|
|
51
|
+
const canReuseHash =
|
|
52
|
+
!rehash &&
|
|
53
|
+
previousEntry &&
|
|
54
|
+
previousEntry.size_bytes === stat.size &&
|
|
55
|
+
previousEntry.mtime_ms === Math.round(stat.mtimeMs) &&
|
|
56
|
+
typeof previousEntry.sha256 === 'string';
|
|
57
|
+
if (canReuseHash) reusedHashCount += 1;
|
|
58
|
+
const sha = canReuseHash ? previousEntry.sha256 : sha256File(abs);
|
|
59
|
+
const rawMetadata = readRawMetadata(abs);
|
|
60
|
+
const sourceId = rawMetadata.source_id || inferSourceId(rel);
|
|
61
|
+
const envelope = sourceId ? readEnvelope(root, vaultCfg, sourceId) : null;
|
|
62
|
+
const text = isTextSource(abs) ? readText(abs, '').slice(0, 30000) : '';
|
|
63
|
+
const sensitivity = envelope?.sensitivity || rawMetadata.sensitivity || pathSensitivity(rel, text, resolver, intel, vaultCfg);
|
|
64
|
+
const compiledTo = inferCompiledTo(envelope, rawMetadata, previousEntry);
|
|
65
|
+
const status = inferStatus(envelope, previousEntry, rawMetadata, compiledTo);
|
|
66
|
+
const assessment = assessHighValue(rel, stat, sensitivity, text);
|
|
67
|
+
const entry = {
|
|
68
|
+
path: rel,
|
|
69
|
+
sha256: sha,
|
|
70
|
+
size_bytes: stat.size,
|
|
71
|
+
mtime_ms: Math.round(stat.mtimeMs),
|
|
72
|
+
source_id: sourceId,
|
|
73
|
+
source_type: envelope?.source_type || rawMetadata.source_type || inferSourceType(rel),
|
|
74
|
+
sensitivity,
|
|
75
|
+
status,
|
|
76
|
+
compiled_to: compiledTo,
|
|
77
|
+
first_seen: previousEntry?.first_seen || now,
|
|
78
|
+
last_seen: now,
|
|
79
|
+
last_changed: previousEntry?.sha256 && previousEntry.sha256 !== sha ? now : previousEntry?.last_changed || now,
|
|
80
|
+
high_value_score: assessment.score,
|
|
81
|
+
high_value_reasons: assessment.reasons,
|
|
82
|
+
};
|
|
83
|
+
nextFiles[rel] = entry;
|
|
84
|
+
|
|
85
|
+
if (!previousEntry) newItems.push(entry);
|
|
86
|
+
else if (previousEntry.sha256 !== sha) changedItems.push(entry);
|
|
87
|
+
else unchangedItems.push(entry);
|
|
88
|
+
|
|
89
|
+
if (!isCompiled(entry) && entry.high_value_score >= 2) highValueCandidates.push(entry);
|
|
90
|
+
}
|
|
91
|
+
|
|
92
|
+
const removed = Object.keys(previous.files || {}).filter((rel) => !nextFiles[rel]);
|
|
93
|
+
const nextManifest = {
|
|
94
|
+
version: 1,
|
|
95
|
+
updated_at: now,
|
|
96
|
+
scope,
|
|
97
|
+
generated_by: 'source_digest_gen.mjs',
|
|
98
|
+
policy: 'Tracks raw source fingerprints so unchanged files can be skipped and high-value sources can be selected for deep compile.',
|
|
99
|
+
files: nextFiles,
|
|
100
|
+
};
|
|
101
|
+
|
|
102
|
+
if (write) {
|
|
103
|
+
ensureDir(path.dirname(manifestAbs));
|
|
104
|
+
atomicWriteJson(manifestAbs, nextManifest);
|
|
105
|
+
}
|
|
106
|
+
|
|
107
|
+
const sortedHighValue = highValueCandidates
|
|
108
|
+
.sort((a, b) => b.high_value_score - a.high_value_score || b.size_bytes - a.size_bytes)
|
|
109
|
+
.slice(0, top)
|
|
110
|
+
.map(summarizeEntry);
|
|
111
|
+
|
|
112
|
+
const payload = {
|
|
113
|
+
ok: true,
|
|
114
|
+
command: 'source-digest',
|
|
115
|
+
durable_write: Boolean(write),
|
|
116
|
+
manifest: manifestRel,
|
|
117
|
+
scope,
|
|
118
|
+
write,
|
|
119
|
+
scanned_count: files.length,
|
|
120
|
+
reused_hash_count: reusedHashCount,
|
|
121
|
+
new_count: newItems.length,
|
|
122
|
+
changed_count: changedItems.length,
|
|
123
|
+
unchanged_count: unchangedItems.length,
|
|
124
|
+
removed_count: removed.length,
|
|
125
|
+
compiled_count: Object.values(nextFiles).filter(isCompiled).length,
|
|
126
|
+
uncompiled_count: Object.values(nextFiles).filter((entry) => !isCompiled(entry)).length,
|
|
127
|
+
high_value_candidate_count: highValueCandidates.length,
|
|
128
|
+
high_value_candidates: sortedHighValue,
|
|
129
|
+
removed: removed.slice(0, top),
|
|
130
|
+
};
|
|
131
|
+
|
|
132
|
+
if (args.json) return { json: true, payload };
|
|
133
|
+
return { text: render(payload) };
|
|
134
|
+
}
|
|
135
|
+
|
|
136
|
+
function scanSourceFiles(root, scope, vaultCfg) {
|
|
137
|
+
const scopeAbs = path.join(root, scope);
|
|
138
|
+
if (!fs.existsSync(scopeAbs)) return [];
|
|
139
|
+
return walkFiles(scopeAbs, { includeRaw: true, maxFiles: 200000 })
|
|
140
|
+
.filter((abs) => {
|
|
141
|
+
const rel = relPath(root, abs);
|
|
142
|
+
if (rel.startsWith(`${vaultCfg.raw_inbox_dir}/`)) return false;
|
|
143
|
+
if (rel.includes('/.git/') || rel.includes('/node_modules/')) return false;
|
|
144
|
+
if (rel.endsWith('.json') && rel.includes('/_inbox/')) return false;
|
|
145
|
+
return true;
|
|
146
|
+
})
|
|
147
|
+
.sort();
|
|
148
|
+
}
|
|
149
|
+
|
|
150
|
+
function sha256File(abs) {
|
|
151
|
+
return crypto.createHash('sha256').update(fs.readFileSync(abs)).digest('hex');
|
|
152
|
+
}
|
|
153
|
+
|
|
154
|
+
function isTextSource(abs) {
|
|
155
|
+
const ext = path.extname(abs).toLowerCase();
|
|
156
|
+
return ext === '.md' || ext === '.txt';
|
|
157
|
+
}
|
|
158
|
+
|
|
159
|
+
function inferSourceId(rel) {
|
|
160
|
+
const match = rel.match(/\b(raw-\d{8}-\d{3})\b/);
|
|
161
|
+
return match ? match[1] : '';
|
|
162
|
+
}
|
|
163
|
+
|
|
164
|
+
function readRawMetadata(abs) {
|
|
165
|
+
if (!isTextSource(abs)) return {};
|
|
166
|
+
const text = readText(abs, '').slice(0, 12000);
|
|
167
|
+
const fm = text.match(/^---\r?\n([\s\S]*?)\r?\n---/);
|
|
168
|
+
if (!fm) return {};
|
|
169
|
+
return parseSimpleFrontmatter(fm[1]);
|
|
170
|
+
}
|
|
171
|
+
|
|
172
|
+
function parseSimpleFrontmatter(body) {
|
|
173
|
+
const meta = {};
|
|
174
|
+
let currentKey = '';
|
|
175
|
+
for (const rawLine of body.split(/\r?\n/)) {
|
|
176
|
+
const line = rawLine.trimEnd();
|
|
177
|
+
const trimmed = line.trim();
|
|
178
|
+
if (!trimmed || trimmed.startsWith('#')) continue;
|
|
179
|
+
const listMatch = line.match(/^\s+-\s*(.+)$/);
|
|
180
|
+
if (listMatch && currentKey) {
|
|
181
|
+
if (!Array.isArray(meta[currentKey])) meta[currentKey] = [];
|
|
182
|
+
meta[currentKey].push(cleanScalar(listMatch[1]));
|
|
183
|
+
continue;
|
|
184
|
+
}
|
|
185
|
+
const keyMatch = line.match(/^([A-Za-z0-9_-]+):\s*(.*)$/);
|
|
186
|
+
if (!keyMatch) continue;
|
|
187
|
+
currentKey = keyMatch[1];
|
|
188
|
+
const rawValue = keyMatch[2].trim();
|
|
189
|
+
if (!rawValue) meta[currentKey] = [];
|
|
190
|
+
else if (rawValue.startsWith('[') && rawValue.endsWith(']')) meta[currentKey] = rawValue.slice(1, -1).split(',').map(cleanScalar).filter(Boolean);
|
|
191
|
+
else meta[currentKey] = cleanScalar(rawValue);
|
|
192
|
+
}
|
|
193
|
+
return meta;
|
|
194
|
+
}
|
|
195
|
+
|
|
196
|
+
function cleanScalar(value) {
|
|
197
|
+
const text = String(value).trim();
|
|
198
|
+
if ((text.startsWith('"') && text.endsWith('"')) || (text.startsWith("'") && text.endsWith("'"))) return text.slice(1, -1);
|
|
199
|
+
return text;
|
|
200
|
+
}
|
|
201
|
+
|
|
202
|
+
function readEnvelope(root, vaultCfg, sourceId) {
|
|
203
|
+
return readJsonSafe(path.join(root, `${vaultCfg.raw_inbox_dir}/${sourceId}.json`), null);
|
|
204
|
+
}
|
|
205
|
+
|
|
206
|
+
function inferSourceType(rel) {
|
|
207
|
+
if (rel.includes('/web/videos/')) return 'video';
|
|
208
|
+
if (rel.includes('/web/articles/')) return 'article';
|
|
209
|
+
if (rel.includes('/web/research/')) return 'paper';
|
|
210
|
+
if (rel.includes('/web/podcasts/')) return 'podcast';
|
|
211
|
+
if (rel.includes('/web/books/')) return 'book';
|
|
212
|
+
if (rel.includes('/files/')) return 'file';
|
|
213
|
+
if (rel.includes('/screenshots/')) return 'screenshot';
|
|
214
|
+
return 'raw';
|
|
215
|
+
}
|
|
216
|
+
|
|
217
|
+
// Path-based sensitivity fallback (only reached when neither the envelope nor the
|
|
218
|
+
// file frontmatter sets sensitivity). Uses the W-A label resolver for the PRIVATE
|
|
219
|
+
// decision (frontmatter + _area.md baseline + boundary path markers) and the
|
|
220
|
+
// registry's per-area baseline for INTERNAL, plus generic business markers. No
|
|
221
|
+
// hardcoded area names, so no vault identifier enters the engine.
|
|
222
|
+
function pathSensitivity(rel, text, resolver, intel, vaultCfg) {
|
|
223
|
+
if (resolver.sensitivityFor(rel, text) === 'private') return 'private';
|
|
224
|
+
const areaMatch = rel.match(new RegExp(`^${vaultCfg.areas_dir.replace(/[.*+?^${}()|[\]\\]/g, '\\$&')}/([^/]+)/`));
|
|
225
|
+
if (areaMatch) {
|
|
226
|
+
const baseline = String((intel.sensitivity && intel.sensitivity[areaMatch[1]]) || '').toLowerCase();
|
|
227
|
+
if (/private/.test(baseline)) return 'private';
|
|
228
|
+
if (/internal/.test(baseline)) return 'internal';
|
|
229
|
+
}
|
|
230
|
+
if (/\b(internal|strategy|governance|partner|foundation|roadmap)\b/.test(rel.toLowerCase())) return 'internal';
|
|
231
|
+
return 'public';
|
|
232
|
+
}
|
|
233
|
+
|
|
234
|
+
function inferCompiledTo(envelope, rawMetadata, previousEntry) {
|
|
235
|
+
if (envelope?.compiled_to?.length) return envelope.compiled_to;
|
|
236
|
+
if (Array.isArray(rawMetadata.compiled_to) && rawMetadata.compiled_to.length) return rawMetadata.compiled_to;
|
|
237
|
+
if (typeof rawMetadata.compiled_to === 'string' && rawMetadata.compiled_to.trim()) return [rawMetadata.compiled_to.trim()];
|
|
238
|
+
return previousEntry?.compiled_to || [];
|
|
239
|
+
}
|
|
240
|
+
|
|
241
|
+
function normalizeStatus(status) {
|
|
242
|
+
return typeof status === 'string' ? status.trim().replace(/^["']|["']$/g, '') : '';
|
|
243
|
+
}
|
|
244
|
+
|
|
245
|
+
function inferStatus(envelope, previousEntry, rawMetadata, compiledTo) {
|
|
246
|
+
const envelopeStatus = normalizeStatus(envelope?.status);
|
|
247
|
+
const rawStatus = normalizeStatus(rawMetadata.status);
|
|
248
|
+
const rawIngestStatus = normalizeStatus(rawMetadata.ingest_status);
|
|
249
|
+
if (TERMINAL_STATUSES.has(envelopeStatus)) return envelopeStatus;
|
|
250
|
+
if (TERMINAL_STATUSES.has(rawIngestStatus)) return rawIngestStatus;
|
|
251
|
+
if (TERMINAL_STATUSES.has(rawStatus)) return rawStatus;
|
|
252
|
+
if ((compiledTo || []).length) return 'compiled';
|
|
253
|
+
if (envelopeStatus) return envelopeStatus;
|
|
254
|
+
if (rawIngestStatus && rawIngestStatus !== 'inbox') return rawIngestStatus;
|
|
255
|
+
if (rawStatus) return rawStatus;
|
|
256
|
+
return previousEntry?.status || 'discovered';
|
|
257
|
+
}
|
|
258
|
+
|
|
259
|
+
function isCompiled(entry) {
|
|
260
|
+
return TERMINAL_STATUSES.has(entry.status) || (entry.compiled_to || []).length > 0;
|
|
261
|
+
}
|
|
262
|
+
|
|
263
|
+
function assessHighValue(rel, stat, sensitivity, text) {
|
|
264
|
+
const reasons = [];
|
|
265
|
+
let score = 0;
|
|
266
|
+
const ext = path.extname(rel).toLowerCase();
|
|
267
|
+
const lowerPath = rel.toLowerCase();
|
|
268
|
+
if (HIGH_VALUE_EXTS.has(ext)) { score += 1; reasons.push(`structured or long-form source: ${ext}`); }
|
|
269
|
+
if (stat.size > 1_000_000) { score += 1; reasons.push('larger than 1MB'); }
|
|
270
|
+
if (stat.size > 10_000_000) { score += 1; reasons.push('larger than 10MB'); }
|
|
271
|
+
if (['private', 'internal'].includes(sensitivity)) { score += 1; reasons.push('sensitive or internal source'); }
|
|
272
|
+
const haystack = `${lowerPath}\n${text.toLowerCase()}`;
|
|
273
|
+
const patterns = [
|
|
274
|
+
['investment', /investment|investor|valuation|revenue|model|dd|due diligence|term sheet|투자|밸류|매출|실사|텀시트/],
|
|
275
|
+
['legal or contract', /contract|agreement|legal|license|tax|loan|real estate|계약|법무|세금|대출|부동산|등기/],
|
|
276
|
+
['strategy or roadmap', /strategy|roadmap|milestone|checkpoint|current status|전략|로드맵|마일스톤|체크포인트|현황/],
|
|
277
|
+
['health or family administration', /health|medical|family|parents|credentials|건강|의료|가족|부모|증명서/],
|
|
278
|
+
];
|
|
279
|
+
for (const [label, regex] of patterns) {
|
|
280
|
+
if (regex.test(haystack)) { score += 1; reasons.push(label); }
|
|
281
|
+
}
|
|
282
|
+
if (text.length > 6000) { score += 1; reasons.push('long markdown source'); }
|
|
283
|
+
if (text.length > 20000) { score += 1; reasons.push('very long markdown source'); }
|
|
284
|
+
return { score, reasons: [...new Set(reasons)] };
|
|
285
|
+
}
|
|
286
|
+
|
|
287
|
+
function summarizeEntry(entry) {
|
|
288
|
+
return {
|
|
289
|
+
path: entry.path,
|
|
290
|
+
source_id: entry.source_id,
|
|
291
|
+
source_type: entry.source_type,
|
|
292
|
+
sensitivity: entry.sensitivity,
|
|
293
|
+
status: entry.status,
|
|
294
|
+
high_value_score: entry.high_value_score,
|
|
295
|
+
high_value_reasons: entry.high_value_reasons,
|
|
296
|
+
suggested_action: 'Deep Compile selected source before promoting official memory.',
|
|
297
|
+
};
|
|
298
|
+
}
|
|
299
|
+
|
|
300
|
+
function render(result) {
|
|
301
|
+
const lines = ['# Source Digest', ''];
|
|
302
|
+
lines.push(`- Manifest: ${result.manifest}`);
|
|
303
|
+
lines.push(`- Updated: ${result.write ? 'yes' : 'no, dry report only'}`);
|
|
304
|
+
lines.push(`- Scanned: ${result.scanned_count}`);
|
|
305
|
+
lines.push(`- Reused hashes (size+mtime unchanged): ${result.reused_hash_count}`);
|
|
306
|
+
lines.push(`- New: ${result.new_count}`);
|
|
307
|
+
lines.push(`- Changed: ${result.changed_count}`);
|
|
308
|
+
lines.push(`- Uncompiled: ${result.uncompiled_count}`);
|
|
309
|
+
lines.push(`- High-value candidates: ${result.high_value_candidate_count}`);
|
|
310
|
+
return lines.join('\n');
|
|
311
|
+
}
|
|
@@ -0,0 +1,105 @@
|
|
|
1
|
+
// `stage` command + stageAndPromote primitive (W-B). The tool-enforced "a secret
|
|
2
|
+
// never reaches canonical" guarantee: content is written to .neurain-staging
|
|
3
|
+
// first (excluded from search/label walks), scanned fail-closed for secrets, and
|
|
4
|
+
// ONLY promoted to its canonical target by an atomic rename if clean. On a high-
|
|
5
|
+
// confidence hit it stays in staging (never indexed), the canonical target is
|
|
6
|
+
// untouched, and the command exits 2. Every B4 durable canonical write routes
|
|
7
|
+
// through here. Faithful port of the vault neurain-stage + wrap-intel
|
|
8
|
+
// stageAndPromote.
|
|
9
|
+
import fs from 'node:fs';
|
|
10
|
+
import path from 'node:path';
|
|
11
|
+
import { absPath, ensureDir, readText, safeResolve, sha256, timestamp } from './fs.mjs';
|
|
12
|
+
import { atomicWriteText, readJsonlRows } from './durable.mjs';
|
|
13
|
+
import { isBlocking, scanSecrets, summarizeHits } from './secret_scan.mjs';
|
|
14
|
+
|
|
15
|
+
export const STAGING_DIR = '.neurain-staging';
|
|
16
|
+
export const PROMOTION_LEDGER = `${STAGING_DIR}/.promotions.jsonl`;
|
|
17
|
+
const LEDGER_CAP = 5000;
|
|
18
|
+
|
|
19
|
+
// Stage content, scan it, and atomically promote it to `relTarget` only if clean.
|
|
20
|
+
// A blocking secret (unless allowSecret) leaves the content in staging and the
|
|
21
|
+
// canonical target untouched.
|
|
22
|
+
export function stageAndPromote(root, relTarget, content, { allowSecret = false } = {}) {
|
|
23
|
+
const safeRel = String(relTarget).replace(/^\/+/, '');
|
|
24
|
+
const stagedRel = `${STAGING_DIR}/${safeRel}`;
|
|
25
|
+
const stagedAbs = safeResolve(root, stagedRel);
|
|
26
|
+
atomicWriteText(stagedAbs, content); // land in excluded staging first
|
|
27
|
+
|
|
28
|
+
const hits = scanSecrets(content);
|
|
29
|
+
if (isBlocking(hits) && !allowSecret) {
|
|
30
|
+
return { promoted: false, blocked: 'secret', staged_path: stagedRel, canonical_path: safeRel, hits, summary: summarizeHits(hits) };
|
|
31
|
+
}
|
|
32
|
+
|
|
33
|
+
const canonAbs = safeResolve(root, safeRel);
|
|
34
|
+
ensureDir(path.dirname(canonAbs));
|
|
35
|
+
fs.renameSync(stagedAbs, canonAbs); // atomic promote (same filesystem under root)
|
|
36
|
+
|
|
37
|
+
// Record proof of promotion (content sha) so a later check can prove this
|
|
38
|
+
// canonical file was scanned before it reached the indexed tree.
|
|
39
|
+
const ledgerAbs = safeResolve(root, PROMOTION_LEDGER);
|
|
40
|
+
ensureDir(path.dirname(ledgerAbs));
|
|
41
|
+
fs.appendFileSync(ledgerAbs, `${JSON.stringify({ path: safeRel, sha256: sha256(content), promoted_at: timestamp() })}\n`);
|
|
42
|
+
try {
|
|
43
|
+
const rows = readJsonlRows(ledgerAbs);
|
|
44
|
+
if (rows.length > LEDGER_CAP) {
|
|
45
|
+
atomicWriteText(ledgerAbs, `${rows.slice(-Math.floor(LEDGER_CAP / 2)).map((r) => JSON.stringify(r)).join('\n')}\n`);
|
|
46
|
+
}
|
|
47
|
+
} catch { /* rotation is best-effort */ }
|
|
48
|
+
return { promoted: true, path: safeRel, hits, summary: summarizeHits(hits) };
|
|
49
|
+
}
|
|
50
|
+
|
|
51
|
+
// Did this canonical file's CURRENT content get promoted via staging (its sha is
|
|
52
|
+
// in the ledger)? Direct writes that bypass the gate return false.
|
|
53
|
+
export function wasStagedFile(root, relTarget) {
|
|
54
|
+
const safeRel = String(relTarget).replace(/^\/+/, '');
|
|
55
|
+
let content;
|
|
56
|
+
try { content = fs.readFileSync(safeResolve(root, safeRel), 'utf8'); } catch { return false; }
|
|
57
|
+
const sha = sha256(content);
|
|
58
|
+
const ledgerAbs = safeResolve(root, PROMOTION_LEDGER);
|
|
59
|
+
if (!fs.existsSync(ledgerAbs)) return false;
|
|
60
|
+
let rows = [];
|
|
61
|
+
try { rows = readJsonlRows(ledgerAbs); } catch { return false; }
|
|
62
|
+
return rows.some((r) => r && r.path === safeRel && r.sha256 === sha);
|
|
63
|
+
}
|
|
64
|
+
|
|
65
|
+
function readStageContent(args, root) {
|
|
66
|
+
if (typeof args.text === 'string') return args.text;
|
|
67
|
+
if (args.file) {
|
|
68
|
+
const target = path.isAbsolute(String(args.file)) ? String(args.file) : safeResolve(root, String(args.file));
|
|
69
|
+
return readText(target, '');
|
|
70
|
+
}
|
|
71
|
+
if (!process.stdin.isTTY) {
|
|
72
|
+
try { return fs.readFileSync(0, 'utf8'); } catch { return ''; }
|
|
73
|
+
}
|
|
74
|
+
return '';
|
|
75
|
+
}
|
|
76
|
+
|
|
77
|
+
export async function stageCommand(args) {
|
|
78
|
+
const root = absPath(args._[0] || args.root || process.cwd());
|
|
79
|
+
const target = String(args.target || '');
|
|
80
|
+
if (!target) {
|
|
81
|
+
process.exitCode = 1;
|
|
82
|
+
return done(args, { ok: false, command: 'stage', durable_write: false, error: 'Missing --target <relative path>.' });
|
|
83
|
+
}
|
|
84
|
+
const content = String(readStageContent(args, root) || '');
|
|
85
|
+
if (!content) {
|
|
86
|
+
process.exitCode = 1;
|
|
87
|
+
return done(args, { ok: false, command: 'stage', durable_write: false, error: 'Missing content. Pass --text, --file, or stdin.' });
|
|
88
|
+
}
|
|
89
|
+
|
|
90
|
+
const res = stageAndPromote(root, target, content, { allowSecret: Boolean(args['allow-secret']) });
|
|
91
|
+
process.exitCode = res.promoted ? 0 : 2;
|
|
92
|
+
return done(args, { ok: res.promoted, command: 'stage', durable_write: Boolean(res.promoted), ...res });
|
|
93
|
+
}
|
|
94
|
+
|
|
95
|
+
function done(args, payload) {
|
|
96
|
+
if (args.json) return { json: true, payload };
|
|
97
|
+
if (payload.error) return { text: `# Neurain Stage\n\n- ${payload.error}` };
|
|
98
|
+
if (payload.promoted) {
|
|
99
|
+
return { text: `✓ promoted to ${payload.path}${payload.hits.length ? ` (non-blocking notes: ${payload.summary})` : ''}` };
|
|
100
|
+
}
|
|
101
|
+
const lines = [`⛔ BLOCKED (secret): ${payload.summary}. Content held in ${payload.staged_path}; ${payload.canonical_path} NOT written.`];
|
|
102
|
+
for (const h of payload.hits) lines.push(` [${h.confidence}] ${h.type}: ${h.sample}`);
|
|
103
|
+
lines.push('Redact the source, or pass --allow-secret for a verified false positive.');
|
|
104
|
+
return { text: lines.join('\n') };
|
|
105
|
+
}
|
|
@@ -0,0 +1,175 @@
|
|
|
1
|
+
// Read-only session status: boot summary, per-session detail, and an all-sessions
|
|
2
|
+
// list. This is the data-driven core of the vault's session-status tool. It is
|
|
3
|
+
// strictly read-only: it never advances the digest ack pointer and never runs
|
|
4
|
+
// maintenance (--fix). The vault-product presentation layer (command menu,
|
|
5
|
+
// decision guide, optimization lens, localized next-step copy) is intentionally
|
|
6
|
+
// NOT hardcoded here so the engine stays generic; a vault shim composes that on
|
|
7
|
+
// top of this data. lesson_review counts come from the engine's own lessons
|
|
8
|
+
// registry, not the vault's lesson-intel.
|
|
9
|
+
import fs from 'node:fs';
|
|
10
|
+
import path from 'node:path';
|
|
11
|
+
import { absPath, exists } from './fs.mjs';
|
|
12
|
+
import { vaultConfig } from './config.mjs';
|
|
13
|
+
import { areaBriefPath, extractMarkdownSection, freshnessFor, loadSessionState, pendingQueueRows, readJsonl, readTextAt } from './vault_state.mjs';
|
|
14
|
+
import { compileCandidateSummary } from './source_digest.mjs';
|
|
15
|
+
import { computeDigest, renderDigest } from './digest.mjs';
|
|
16
|
+
import { listLessons } from './lessons.mjs';
|
|
17
|
+
|
|
18
|
+
function summarizeSession(root, vaultCfg, session, { staleDays, now }) {
|
|
19
|
+
const handoffRel = session.handoff_path || '';
|
|
20
|
+
const handoffText = readTextAt(root, handoffRel);
|
|
21
|
+
const areaBriefRel = session.area ? areaBriefPath(vaultCfg, session.area) : '';
|
|
22
|
+
const areaBriefText = areaBriefRel ? readTextAt(root, areaBriefRel) : '';
|
|
23
|
+
const pendingCount = pendingQueueRows(root, vaultCfg, { sessionId: session.session_id }).length;
|
|
24
|
+
const freshness = freshnessFor(session.last_pulse, staleDays, now);
|
|
25
|
+
return {
|
|
26
|
+
session_id: session.session_id,
|
|
27
|
+
area: session.area,
|
|
28
|
+
scope: session.scope,
|
|
29
|
+
status: session.status,
|
|
30
|
+
sensitivity: session.sensitivity,
|
|
31
|
+
last_pulse: session.last_pulse || '',
|
|
32
|
+
last_flush: session.last_flush || '',
|
|
33
|
+
...freshness,
|
|
34
|
+
pending_count: pendingCount,
|
|
35
|
+
handoff_path: handoffRel,
|
|
36
|
+
handoff_exists: handoffRel ? exists(path.join(root, handoffRel)) : false,
|
|
37
|
+
handoff_lines: handoffText ? handoffText.split(/\r?\n/).length : 0,
|
|
38
|
+
area_brief_path: areaBriefRel,
|
|
39
|
+
area_brief_exists: areaBriefRel ? exists(path.join(root, areaBriefRel)) : false,
|
|
40
|
+
area_brief_latest_notes: extractMarkdownSection(areaBriefText, 'Latest Cross-Session Notes'),
|
|
41
|
+
current_focus: extractMarkdownSection(handoffText, 'Current Focus'),
|
|
42
|
+
next_suggested_action: extractMarkdownSection(handoffText, 'Next Suggested Action'),
|
|
43
|
+
read_first: session.read_first || [],
|
|
44
|
+
};
|
|
45
|
+
}
|
|
46
|
+
|
|
47
|
+
function lessonReview(root) {
|
|
48
|
+
let active = 0;
|
|
49
|
+
try { active = listLessons(root).length; } catch { active = 0; }
|
|
50
|
+
return { active_lessons: active, source: 'engine_lessons' };
|
|
51
|
+
}
|
|
52
|
+
|
|
53
|
+
function bootStatus(root, vaultCfg, state, { staleDays, now }) {
|
|
54
|
+
const sessions = Object.values(state.sessions || {}).map((s) => summarizeSession(root, vaultCfg, s, { staleDays, now }));
|
|
55
|
+
const queueRows = pendingQueueRows(root, vaultCfg);
|
|
56
|
+
const candidates = compileCandidateSummary(root, vaultCfg);
|
|
57
|
+
const freshnessSummary = sessions.reduce((acc, s) => { acc[s.freshness_status] = (acc[s.freshness_status] || 0) + 1; return acc; }, {});
|
|
58
|
+
const activeAreas = [...new Set(sessions.map((s) => s.area).filter(Boolean))].sort();
|
|
59
|
+
const staleSessions = sessions.filter((s) => s.freshness_status === 'stale');
|
|
60
|
+
const recentSessions = [...sessions]
|
|
61
|
+
.sort((a, b) => String(b.last_pulse || '').localeCompare(String(a.last_pulse || '')))
|
|
62
|
+
.slice(0, 5)
|
|
63
|
+
.map((s) => ({ session_id: s.session_id, area: s.area, freshness_status: s.freshness_status, last_pulse: s.last_pulse, pending_count: s.pending_count, area_brief_path: s.area_brief_path }));
|
|
64
|
+
const requiredFiles = vaultCfg.status_required_files.map((rel) => ({ path: rel, exists: exists(path.join(root, rel)) }));
|
|
65
|
+
return {
|
|
66
|
+
ok: requiredFiles.every((f) => f.exists),
|
|
67
|
+
command: 'status',
|
|
68
|
+
durable_write: false,
|
|
69
|
+
mode: 'boot_status',
|
|
70
|
+
required_files: requiredFiles,
|
|
71
|
+
updated_at: state.updated_at || '',
|
|
72
|
+
session_count: sessions.length,
|
|
73
|
+
active_areas: activeAreas,
|
|
74
|
+
freshness_summary: freshnessSummary,
|
|
75
|
+
total_pending_count: queueRows.length,
|
|
76
|
+
pending_requires_confirmation_count: queueRows.filter((r) => r.requires_user_decision || r.sensitivity === 'private' || r.flush_level === 'full').length,
|
|
77
|
+
compile_candidate_count: candidates.count,
|
|
78
|
+
compile_candidates: candidates.examples,
|
|
79
|
+
recent_sessions: recentSessions,
|
|
80
|
+
next_action: queueRows.length > 0
|
|
81
|
+
? 'Review pending candidates before official memory changes.'
|
|
82
|
+
: candidates.count > 0
|
|
83
|
+
? 'Review source candidates before deep reading.'
|
|
84
|
+
: 'Choose a session or area and continue work.',
|
|
85
|
+
stale_guidance: staleSessions.length
|
|
86
|
+
? { count: staleSessions.length, action: 'Not an error. Resume those sessions, or consolidate only when closing them.', examples: staleSessions.slice(0, 3).map((s) => s.session_id) }
|
|
87
|
+
: null,
|
|
88
|
+
lesson_review: lessonReview(root),
|
|
89
|
+
health_maintenance: null,
|
|
90
|
+
health_maintenance_mode: 'read_only',
|
|
91
|
+
note: 'Read-only status. Maintenance (--fix) and the digest ack are vault-side until the session-state write surface migrates.',
|
|
92
|
+
};
|
|
93
|
+
}
|
|
94
|
+
|
|
95
|
+
export async function statusCommand(args) {
|
|
96
|
+
const root = absPath(args._[0] || args.root || process.cwd());
|
|
97
|
+
const vaultCfg = vaultConfig(root);
|
|
98
|
+
const staleDays = Number(args['stale-days'] || 7);
|
|
99
|
+
const now = args.now !== undefined ? Number(args.now) : Date.now();
|
|
100
|
+
let state;
|
|
101
|
+
try {
|
|
102
|
+
state = loadSessionState(root, vaultCfg);
|
|
103
|
+
} catch (error) {
|
|
104
|
+
const payload = { ok: false, command: 'status', durable_write: false, error: error.message };
|
|
105
|
+
return args.json ? { json: true, payload } : { text: `# Neurain status\n\n- ERROR: ${error.message}` };
|
|
106
|
+
}
|
|
107
|
+
const sessionId = args['session-id'];
|
|
108
|
+
|
|
109
|
+
if (!sessionId && !args.all) {
|
|
110
|
+
const payload = bootStatus(root, vaultCfg, state, { staleDays, now });
|
|
111
|
+
if (args.json) return { json: true, payload };
|
|
112
|
+
return {
|
|
113
|
+
text: [
|
|
114
|
+
'# Neurain Status',
|
|
115
|
+
'',
|
|
116
|
+
`- OK: ${payload.ok ? 'yes' : 'needs attention'}`,
|
|
117
|
+
`- Sessions: ${payload.session_count} | Areas: ${payload.active_areas.join(', ') || 'none'}`,
|
|
118
|
+
`- Pending: ${payload.total_pending_count}${payload.pending_requires_confirmation_count ? ` (confirmation needed: ${payload.pending_requires_confirmation_count})` : ''}`,
|
|
119
|
+
payload.compile_candidate_count ? `- Compile candidates: ${payload.compile_candidate_count}` : '',
|
|
120
|
+
payload.stale_guidance ? `- Stale sessions: ${payload.stale_guidance.count} (not an error)` : '',
|
|
121
|
+
`- Next: ${payload.next_action}`,
|
|
122
|
+
].filter(Boolean).join('\n'),
|
|
123
|
+
};
|
|
124
|
+
}
|
|
125
|
+
|
|
126
|
+
if (args.all) {
|
|
127
|
+
const sessions = Object.values(state.sessions || {}).map((s) => summarizeSession(root, vaultCfg, s, { staleDays, now }));
|
|
128
|
+
const payload = { ok: true, command: 'status', durable_write: false, mode: 'all', updated_at: state.updated_at || '', count: sessions.length, sessions };
|
|
129
|
+
if (args.json) return { json: true, payload };
|
|
130
|
+
return {
|
|
131
|
+
text: ['# Session Status', '', `- Sessions: ${payload.count}`, ...sessions.map((s) => `- ${s.session_id}: ${s.status}, freshness ${s.freshness_status}, pending ${s.pending_count}, handoff ${s.handoff_exists ? 'ok' : 'missing'}`)].join('\n'),
|
|
132
|
+
};
|
|
133
|
+
}
|
|
134
|
+
|
|
135
|
+
const session = state.sessions[String(sessionId)];
|
|
136
|
+
if (!session) {
|
|
137
|
+
process.stderr.write(`Unknown session_id: ${sessionId}\n`);
|
|
138
|
+
const payload = { ok: false, command: 'status', durable_write: false, error: `Unknown session_id: ${sessionId}` };
|
|
139
|
+
if (args.json) return { json: true, payload };
|
|
140
|
+
return { text: `# Session Status\n\n- Unknown session_id: ${sessionId}` };
|
|
141
|
+
}
|
|
142
|
+
const sinceLastVisit = computeDigest(root, session.session_id, { now, vaultCfg });
|
|
143
|
+
const payload = {
|
|
144
|
+
ok: true,
|
|
145
|
+
command: 'status',
|
|
146
|
+
durable_write: false,
|
|
147
|
+
mode: 'session',
|
|
148
|
+
...summarizeSession(root, vaultCfg, session, { staleDays, now }),
|
|
149
|
+
queue_items: pendingQueueRows(root, vaultCfg, { sessionId: session.session_id }).map((row) => ({
|
|
150
|
+
source_id: row.source_id || '',
|
|
151
|
+
title: row.title || '',
|
|
152
|
+
flush_level: row.flush_level || '',
|
|
153
|
+
target_layer: row.target_layer || '',
|
|
154
|
+
target_path: row.target_path || '',
|
|
155
|
+
raw_path: row.raw_path || '',
|
|
156
|
+
})),
|
|
157
|
+
since_last_visit: sinceLastVisit,
|
|
158
|
+
lesson_review: lessonReview(root),
|
|
159
|
+
};
|
|
160
|
+
// The engine never acks; a vault shim acks after delivering the rendered text.
|
|
161
|
+
if (args.json) return { json: true, payload };
|
|
162
|
+
return {
|
|
163
|
+
text: [
|
|
164
|
+
'# Session Status',
|
|
165
|
+
'',
|
|
166
|
+
`- Session: ${payload.session_id}`,
|
|
167
|
+
`- Area: ${payload.area}`,
|
|
168
|
+
`- Pending: ${payload.pending_count}`,
|
|
169
|
+
`- Handoff: ${payload.handoff_exists ? payload.handoff_path : 'missing'}`,
|
|
170
|
+
`- Area brief: ${payload.area_brief_exists ? payload.area_brief_path : 'missing'}`,
|
|
171
|
+
`- Freshness: ${payload.freshness_message}`,
|
|
172
|
+
renderDigest(sinceLastVisit) ? `\n${renderDigest(sinceLastVisit)}` : '',
|
|
173
|
+
].filter(Boolean).join('\n'),
|
|
174
|
+
};
|
|
175
|
+
}
|