@descix/cli 1.0.6 → 1.0.7
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/agent-assets/guides/dynamic-site-pattern.md +10 -14
- package/agent-assets/guides/knowledge-only-pattern.md +11 -15
- package/agent-assets/guides/microservice-pattern.md +12 -14
- package/agent-assets/guides/static-site-pattern.md +12 -15
- package/agent-assets/instructions/onboarding-flow.md +4 -4
- package/agent-assets/reference/app-creation.md +36 -101
- package/agent-assets/reference/architecture.md +24 -27
- package/agent-assets/reference/mcp-integration.md +14 -22
- package/agent-assets/reference/workspace-config.md +97 -340
- package/agent-assets/rules/descix_mcp.template.mdc +1 -1
- package/bin/descix.js +137 -75
- package/lib/api-client.js +30 -22
- package/lib/commands/airdrop.js +10 -0
- package/lib/commands/auth.js +16 -2
- package/lib/commands/briefer/index.js +8 -5
- package/lib/commands/config.js +13 -31
- package/lib/commands/corpus.js +161 -54
- package/lib/commands/health.js +116 -21
- package/lib/commands/init.js +17 -45
- package/lib/commands/kb.js +7 -0
- package/lib/core/Chunker.js +1 -1
- package/lib/core/CorpusWalker.js +238 -47
- package/lib/core/ManifestLoader.js +68 -0
- package/lib/core/Syncer.js +147 -46
- package/lib/interactive.js +158 -0
- package/lib/origin.js +3 -3
- package/lib/wallet-file.js +36 -37
- package/lib/wizard/setup.js +104 -22
- package/lib/workspace-config.js +265 -163
- package/lib/workspace-identity.js +0 -1
- package/package.json +2 -2
package/lib/commands/init.js
CHANGED
|
@@ -5,7 +5,6 @@
|
|
|
5
5
|
*/
|
|
6
6
|
|
|
7
7
|
import chalk from 'chalk';
|
|
8
|
-
import readline from 'readline';
|
|
9
8
|
import path from 'path';
|
|
10
9
|
import fs from 'fs/promises';
|
|
11
10
|
import { fileURLToPath } from 'url';
|
|
@@ -17,56 +16,29 @@ import { generateAgentFiles } from '../agent-files.js';
|
|
|
17
16
|
// spelled here is a second derivation of the same fact, and the removed verb this replaced
|
|
18
17
|
// reached a developer who had done everything right.
|
|
19
18
|
import { CANONICAL_KB_SYNC } from './retired-kb-sync.js';
|
|
19
|
+
// "May I prompt?" has ONE OWNER. init does not derive it, and holds no TTY check of its own.
|
|
20
|
+
import { createPromptSession } from '../interactive.js';
|
|
20
21
|
|
|
21
22
|
const __filename = fileURLToPath(import.meta.url);
|
|
22
23
|
const __dirname = dirname(__filename);
|
|
23
24
|
|
|
24
|
-
/**
|
|
25
|
-
* Create readline interface for interactive prompts
|
|
26
|
-
*/
|
|
27
|
-
function createPrompt() {
|
|
28
|
-
return readline.createInterface({
|
|
29
|
-
input: process.stdin,
|
|
30
|
-
output: process.stdout
|
|
31
|
-
});
|
|
32
|
-
}
|
|
33
|
-
|
|
34
|
-
/**
|
|
35
|
-
* Ask for text input with optional default
|
|
36
|
-
*/
|
|
37
|
-
function askInput(rl, question, defaultValue = '') {
|
|
38
|
-
return new Promise((resolve) => {
|
|
39
|
-
const prompt = defaultValue ? `${question} [${defaultValue}]: ` : `${question}: `;
|
|
40
|
-
rl.question(prompt, (answer) => {
|
|
41
|
-
resolve(answer.trim() || defaultValue);
|
|
42
|
-
});
|
|
43
|
-
});
|
|
44
|
-
}
|
|
45
|
-
|
|
46
|
-
/**
|
|
47
|
-
* Ask a yes/no question
|
|
48
|
-
*/
|
|
49
|
-
function askYesNo(rl, question, defaultYes = true) {
|
|
50
|
-
return new Promise((resolve) => {
|
|
51
|
-
const suffix = defaultYes ? '(Y/n)' : '(y/N)';
|
|
52
|
-
rl.question(`${question} ${suffix}: `, (answer) => {
|
|
53
|
-
const trimmed = answer.trim().toLowerCase();
|
|
54
|
-
if (trimmed === '') {
|
|
55
|
-
resolve(defaultYes);
|
|
56
|
-
} else {
|
|
57
|
-
resolve(trimmed === 'y' || trimmed === 'yes');
|
|
58
|
-
}
|
|
59
|
-
});
|
|
60
|
-
});
|
|
61
|
-
}
|
|
62
|
-
|
|
63
25
|
/**
|
|
64
26
|
* Run the init command
|
|
65
27
|
* @param {Object} apiClient - API client (optional, for listing communities)
|
|
66
28
|
* @param {Object} options - Command options
|
|
67
29
|
*/
|
|
68
30
|
export async function runInit(apiClient, options = {}) {
|
|
69
|
-
|
|
31
|
+
// The session is GATED AT CONSTRUCTION: under a non-TTY this throws NonInteractiveError,
|
|
32
|
+
// which bin/descix.js's init action prints and exits 1 on. It can no longer hang on an open
|
|
33
|
+
// pipe/FIFO, nor exit 0 writing nothing on /dev/null.
|
|
34
|
+
const rl = createPromptSession({
|
|
35
|
+
what: 'descix init',
|
|
36
|
+
nonInteractiveForm: [
|
|
37
|
+
'descix init has no non-interactive form today. Run it in a terminal.',
|
|
38
|
+
'',
|
|
39
|
+
'Nothing has been changed. An existing .descix/workspace.json is left as it is.'
|
|
40
|
+
]
|
|
41
|
+
});
|
|
70
42
|
const projectPath = options.path ? path.resolve(options.path) : process.cwd();
|
|
71
43
|
|
|
72
44
|
// If --from-invite provided, resolve the invite first to pre-fill context
|
|
@@ -134,7 +106,7 @@ export async function runInit(apiClient, options = {}) {
|
|
|
134
106
|
}
|
|
135
107
|
|
|
136
108
|
if (hasExisting && !options.force) {
|
|
137
|
-
const overwrite = await askYesNo(
|
|
109
|
+
const overwrite = await rl.askYesNo(chalk.yellow('Workspace already initialized. Overwrite?'), false);
|
|
138
110
|
if (!overwrite) {
|
|
139
111
|
console.log(chalk.gray('\nInitialization cancelled.\n'));
|
|
140
112
|
rl.close();
|
|
@@ -160,7 +132,7 @@ export async function runInit(apiClient, options = {}) {
|
|
|
160
132
|
// ignore
|
|
161
133
|
}
|
|
162
134
|
}
|
|
163
|
-
communityId = await
|
|
135
|
+
communityId = await rl.ask(chalk.white('Community ID'));
|
|
164
136
|
if (!communityId) {
|
|
165
137
|
console.log(chalk.red('\n❌ Community ID is required.\n'));
|
|
166
138
|
rl.close();
|
|
@@ -171,7 +143,7 @@ export async function runInit(apiClient, options = {}) {
|
|
|
171
143
|
const defaultAppName = path.basename(projectPath).toLowerCase().replace(/[^a-z0-9]/g, '_');
|
|
172
144
|
let appName = options.app;
|
|
173
145
|
if (!appName) {
|
|
174
|
-
appName = await
|
|
146
|
+
appName = await rl.ask(chalk.white('App name'), defaultAppName);
|
|
175
147
|
if (!appName) {
|
|
176
148
|
console.log(chalk.red('\n❌ App name is required.\n'));
|
|
177
149
|
rl.close();
|
|
@@ -185,7 +157,7 @@ export async function runInit(apiClient, options = {}) {
|
|
|
185
157
|
console.log(chalk.white(` Community: ${communityId}`));
|
|
186
158
|
console.log(chalk.white(` App: ${appName} (${appId})`));
|
|
187
159
|
|
|
188
|
-
const proceed = await askYesNo(
|
|
160
|
+
const proceed = await rl.askYesNo(chalk.white('\nProceed?'), true);
|
|
189
161
|
if (!proceed) {
|
|
190
162
|
console.log(chalk.gray('\nCancelled.\n'));
|
|
191
163
|
rl.close();
|
package/lib/commands/kb.js
CHANGED
|
@@ -22,6 +22,8 @@ import ora from 'ora';
|
|
|
22
22
|
import * as path from 'path';
|
|
23
23
|
import * as fs from 'fs/promises';
|
|
24
24
|
import inquirer from 'inquirer';
|
|
25
|
+
// "May I prompt?" has ONE OWNER. --interactive is an opt-IN, not a guarantee a human is there.
|
|
26
|
+
import { requireInteractive } from '../interactive.js';
|
|
25
27
|
import { WorkspaceConfig } from '../workspace-config.js';
|
|
26
28
|
import { hydrateKb, pushStaging, checkStagingFiles } from '../core/Hydrator.js';
|
|
27
29
|
import { processKb } from '../core/Chunker.js';
|
|
@@ -176,6 +178,11 @@ export async function runKbPush(apiClient, options) {
|
|
|
176
178
|
// Create conflict prompt callback for interactive mode
|
|
177
179
|
const onConflictPrompt = options.interactive ? async (fileName, fileInfo) => {
|
|
178
180
|
spinner.stop();
|
|
181
|
+
requireInteractive({
|
|
182
|
+
what: 'descix kb push --interactive',
|
|
183
|
+
question: `File "${fileName}" exists in Drive. Overwrite or skip?`,
|
|
184
|
+
nonInteractiveForm: ['descix kb push # without --interactive, conflicts resolve without a prompt']
|
|
185
|
+
});
|
|
179
186
|
const { action } = await inquirer.prompt([{
|
|
180
187
|
type: 'list',
|
|
181
188
|
name: 'action',
|
package/lib/core/Chunker.js
CHANGED
|
@@ -938,7 +938,7 @@ export async function processKb(config, options = {}) {
|
|
|
938
938
|
try {
|
|
939
939
|
await fs.access(srcDir);
|
|
940
940
|
} catch {
|
|
941
|
-
throw new Error(`Source directory not found: ${srcDir}\nRun "descix
|
|
941
|
+
throw new Error(`Source directory not found: ${srcDir}\nRun "descix drive pull" first.`);
|
|
942
942
|
}
|
|
943
943
|
|
|
944
944
|
await fs.mkdir(chunksDir, { recursive: true });
|
package/lib/core/CorpusWalker.js
CHANGED
|
@@ -7,16 +7,19 @@
|
|
|
7
7
|
* Architecture:
|
|
8
8
|
* - Files stay in place — no copying or staging
|
|
9
9
|
* - Git blob SHAs provide versioning (same content = same SHA)
|
|
10
|
-
* -
|
|
10
|
+
* - GIT IS THE ONLY SOURCE OF TRUTH FOR WHAT IS A CORPUS SOURCE: a file must be TRACKED
|
|
11
|
+
* AT THE MANIFEST'S REF or it is refused by name. There is no working-tree fallback.
|
|
11
12
|
* - Chunk IDs are content-addressed: {app_id}:{kb_name}:{blob_sha}:{chunk_idx}
|
|
12
13
|
*/
|
|
13
14
|
|
|
14
15
|
import * as fs from 'fs/promises';
|
|
16
|
+
import * as fsSync from 'fs';
|
|
15
17
|
import * as path from 'path';
|
|
16
18
|
import { execSync } from 'child_process';
|
|
17
19
|
import { minimatch } from 'minimatch';
|
|
18
|
-
// The source-ref default
|
|
19
|
-
|
|
20
|
+
// The source-ref default and "which repository owns this file" are both owned by
|
|
21
|
+
// ManifestLoader — consume them, never re-default and never re-derive.
|
|
22
|
+
import { DEFAULT_SOURCE_REF, resolveOwningRepo } from './ManifestLoader.js';
|
|
20
23
|
|
|
21
24
|
/**
|
|
22
25
|
* File extensions that are processable for RAG chunking.
|
|
@@ -60,42 +63,132 @@ function isIgnored(relativePath, syncignore) {
|
|
|
60
63
|
return false;
|
|
61
64
|
}
|
|
62
65
|
|
|
66
|
+
const SHA40 = /^[0-9a-f]{40}$/i;
|
|
67
|
+
|
|
63
68
|
/**
|
|
64
|
-
*
|
|
65
|
-
*
|
|
66
|
-
*
|
|
69
|
+
* Explain WHICH WAY a path failed the "tracked at ref" predicate. Purely diagnostic —
|
|
70
|
+
* it never decides anything, so there is still exactly one decision site. Each branch
|
|
71
|
+
* ends in an instruction the operator can act on without further investigation.
|
|
67
72
|
*
|
|
68
|
-
* @
|
|
69
|
-
* @param {string} workspaceRelativePath - Path relative to workspace root
|
|
70
|
-
* @param {string} ref - Git ref (branch/tag)
|
|
71
|
-
* @param {string} workspaceRoot - Workspace root directory
|
|
72
|
-
* @returns {string} Git blob SHA (40 hex chars)
|
|
73
|
+
* @returns {string} the reason string embedded in the aggregated refusal
|
|
73
74
|
*/
|
|
74
|
-
function
|
|
75
|
+
function diagnoseUntracked(repoRelativePath, ref, repoRoot, repoLabel) {
|
|
76
|
+
// Every diagnostic below runs IN THE OWNING REPOSITORY. Running them at the ambient
|
|
77
|
+
// workspace root is what produced the maximally wrong advice this gate was blocked for
|
|
78
|
+
// ("UNTRACKED — commit it on main" about content that was already committed, and
|
|
79
|
+
// "IGNORED BY GIT" about a sibling repo the SUPERREPO ignores but which tracks it fine).
|
|
80
|
+
const where = `in ${repoLabel}`;
|
|
81
|
+
|
|
82
|
+
// (c) Present at HEAD but absent at THIS ref — checked first, because .gitignore
|
|
83
|
+
// patterns do not apply to tracked files and would misdiagnose this case.
|
|
75
84
|
try {
|
|
76
|
-
|
|
77
|
-
|
|
78
|
-
|
|
79
|
-
|
|
85
|
+
execSync(`git rev-parse "HEAD:${repoRelativePath}"`,
|
|
86
|
+
{ cwd: repoRoot, encoding: 'utf-8', stdio: ['pipe', 'pipe', 'pipe'] });
|
|
87
|
+
return `present at HEAD but ABSENT AT REF "${ref}" ${where} — the manifest pins a ref ` +
|
|
88
|
+
`that predates this file; commit it on "${ref}" or point the source at the right ref`;
|
|
89
|
+
} catch { /* not at HEAD either — fall through */ }
|
|
90
|
+
|
|
91
|
+
// (a) Deliberately excluded from the repository. The measured defect class.
|
|
92
|
+
try {
|
|
93
|
+
const hit = execSync(`git check-ignore -v "${repoRelativePath}"`,
|
|
94
|
+
{ cwd: repoRoot, encoding: 'utf-8', stdio: ['pipe', 'pipe', 'pipe'] }
|
|
95
|
+
).trim().split('\n')[0];
|
|
96
|
+
if (hit) {
|
|
97
|
+
return `IGNORED BY GIT (${hit.split('\t')[0]}) ${where} and not tracked at ref ` +
|
|
98
|
+
`"${ref}" — content the repository deliberately excludes must never become a ` +
|
|
99
|
+
`citable KB source; remove this source from the manifest`;
|
|
100
|
+
}
|
|
101
|
+
} catch { /* not ignored — fall through */ }
|
|
102
|
+
|
|
103
|
+
// (b) Simply never committed.
|
|
104
|
+
return `UNTRACKED at ref "${ref}" ${where} — the file exists on disk but that repository ` +
|
|
105
|
+
`does not track it; commit it on "${ref}", or remove this source from the manifest`;
|
|
106
|
+
}
|
|
107
|
+
|
|
108
|
+
/**
|
|
109
|
+
* THE SOURCE GATE — the single owner of "is this file a legitimate corpus source".
|
|
110
|
+
*
|
|
111
|
+
* A source is legitimate iff GIT TRACKS IT AT THE MANIFEST'S REF. That one predicate is
|
|
112
|
+
* the whole rule, and it is deliberately not split by WHY a path fails it: gitignored
|
|
113
|
+
* scratch, never-committed, and absent-at-an-older-ref are all "not tracked at ref".
|
|
114
|
+
* Splitting them would re-derive "what is a corpus source" in a second place, and the
|
|
115
|
+
* copies would drift.
|
|
116
|
+
*
|
|
117
|
+
* THE WORKING TREE IS NEVER CONSULTED. Reading a sha from disk when the ref lookup failed
|
|
118
|
+
* is the measured defect: it served content git was explicitly told to exclude as a
|
|
119
|
+
* citable KB source (corpus:04406066... = `.claude/agent-memory/evp-descix/qa-pairs.jsonl`,
|
|
120
|
+
* excluded by .gitignore, chunked and served anyway). That path is deleted, not fenced
|
|
121
|
+
* behind a flag, and there is nothing that restores it — SUPER-DRY, CEO-D-2026-07-26.
|
|
122
|
+
*
|
|
123
|
+
* THE REF IS RESOLVED IN THE REPOSITORY THAT OWNS THE FILE, never in an ambient root. A
|
|
124
|
+
* workspace spans several repositories (superrepo, submodules, ignored sibling checkouts)
|
|
125
|
+
* and a ref names a branch in exactly one of them. Asking the superrepo about a file inside
|
|
126
|
+
* a submodule asks the wrong repository a question it cannot answer — it holds only a
|
|
127
|
+
* gitlink commit — and the honest-looking "not tracked" that comes back is FALSE. That was
|
|
128
|
+
* measured on this very branch: 596 committed files refused, the two largest live KBs
|
|
129
|
+
* stopped, with advice telling the operator to commit content already committed.
|
|
130
|
+
* "Which repository owns this file" has ONE owner: ManifestLoader::resolveOwningRepo.
|
|
131
|
+
*
|
|
132
|
+
* Returns a REASON rather than throwing so the caller can aggregate every offender into
|
|
133
|
+
* one refusal instead of failing on the first (mirroring CorpusDenyLint's report/assert
|
|
134
|
+
* split — the established refusal shape in this codebase).
|
|
135
|
+
*
|
|
136
|
+
* @param {string} absoluteFilePath - Absolute path to the file under test
|
|
137
|
+
* @param {string} ref - Git ref (branch/tag) the manifest source named
|
|
138
|
+
* @param {string} [workspaceRoot] - Only to label paths in messages; NEVER a resolution root
|
|
139
|
+
* @returns {{sha: string|null, reason: string|null, repoRoot: string|null}}
|
|
140
|
+
*/
|
|
141
|
+
export function resolveTrackedBlobSha(absoluteFilePath, ref, workspaceRoot = null) {
|
|
142
|
+
const owner = resolveOwningRepo(absoluteFilePath);
|
|
143
|
+
if (!owner) {
|
|
144
|
+
return {
|
|
145
|
+
sha: null, repoRoot: null,
|
|
146
|
+
reason: `NOT INSIDE ANY GIT REPOSITORY — no repository owns this path, so ref "${ref}" ` +
|
|
147
|
+
`cannot be resolved for it and nothing can vouch for its content; remove this ` +
|
|
148
|
+
`source from the manifest, or put the content in a repository and commit it`
|
|
149
|
+
};
|
|
150
|
+
}
|
|
151
|
+
const { repoRoot, repoRelativePath } = owner;
|
|
152
|
+
// How the owning repo is named in messages: relative to the workspace when we have one,
|
|
153
|
+
// so the operator reads "DeSciX/DeSciX_Cloud" rather than an absolute machine path.
|
|
154
|
+
// `repoRoot` is a REAL path (git resolves symlinks), so the workspace root must be one
|
|
155
|
+
// too or the subtraction escapes upward into `../../..`.
|
|
156
|
+
let workspaceReal = workspaceRoot;
|
|
157
|
+
try { if (workspaceRoot) workspaceReal = fsSync.realpathSync(workspaceRoot); } catch { /* keep as given */ }
|
|
158
|
+
const repoLabel = workspaceRoot ? (path.relative(workspaceReal, repoRoot) || '.') : repoRoot;
|
|
159
|
+
|
|
160
|
+
let sha;
|
|
161
|
+
try {
|
|
162
|
+
sha = execSync(
|
|
163
|
+
`git rev-parse "${ref}:${repoRelativePath}"`,
|
|
164
|
+
{ cwd: repoRoot, encoding: 'utf-8', stdio: ['pipe', 'pipe', 'pipe'] }
|
|
80
165
|
).trim();
|
|
81
|
-
return sha;
|
|
82
166
|
} catch {
|
|
83
|
-
//
|
|
84
|
-
//
|
|
85
|
-
|
|
86
|
-
|
|
87
|
-
|
|
88
|
-
|
|
89
|
-
|
|
90
|
-
|
|
91
|
-
|
|
92
|
-
|
|
93
|
-
|
|
94
|
-
}
|
|
167
|
+
// ONE PREDICATE ("tracked at ref, in the owning repo"), FOUR DIAGNOSTICS. The decision
|
|
168
|
+
// has already been made above — everything below only explains WHICH WAY the path
|
|
169
|
+
// failed, so DRY holds and each population gets a self-curing instruction.
|
|
170
|
+
return { sha: null, repoRoot, reason: diagnoseUntracked(repoRelativePath, ref, repoRoot, repoLabel) };
|
|
171
|
+
}
|
|
172
|
+
|
|
173
|
+
if (!SHA40.test(sha)) {
|
|
174
|
+
return {
|
|
175
|
+
sha: null, repoRoot,
|
|
176
|
+
reason: `git rev-parse "${ref}:${repoRelativePath}" in ${repoLabel} returned "${sha}", which is not a blob sha`
|
|
177
|
+
};
|
|
95
178
|
}
|
|
179
|
+
return { sha, repoRoot, reason: null };
|
|
96
180
|
}
|
|
97
181
|
|
|
98
|
-
|
|
182
|
+
/**
|
|
183
|
+
* What this gate compares, catches, and does NOT read — printed with the verdict so a
|
|
184
|
+
* reader of a GREEN sees where the green stops, in the gate's own words.
|
|
185
|
+
*/
|
|
186
|
+
export const SOURCE_GATE_BOUNDARY = Object.freeze({
|
|
187
|
+
compares: 'each walked file against `git rev-parse <manifest source ref>:<path relative to THE REPOSITORY THAT OWNS THE FILE>`, where the owner is the nearest enclosing git repository (ManifestLoader::resolveOwningRepo) — the superrepo, a submodule, or an ignored sibling checkout',
|
|
188
|
+
catches: 'any corpus source not tracked at its ref BY ITS OWN REPOSITORY — gitignored scratch, never-committed files, paths absent from an older ref, and paths inside no repository at all',
|
|
189
|
+
does_not_read: 'file CONTENT (see CorpusDenyLint for content/path deny-classes), Pinecone live state, and cross-repo `repo:` sources (refused separately by resolveSourceProvenance). It also does NOT check that a submodule\'s ref matches the commit the superrepo pins at its gitlink: a source is walked at the ref it names in the repository that owns it, so content committed on that branch but not yet pinned by the superrepo is accepted.',
|
|
190
|
+
runs_automatically: 'yes — inside walkCorpus, so every `descix kb corpus sync` passes through it; there is no flag that skips it'
|
|
191
|
+
});
|
|
99
192
|
|
|
100
193
|
/**
|
|
101
194
|
* Resolve ONE source's provenance: which repository, which ref, and the exact commit synced.
|
|
@@ -117,33 +210,62 @@ const SHA40 = /^[0-9a-f]{40}$/i;
|
|
|
117
210
|
export async function resolveSourceProvenance(source, workspaceRoot) {
|
|
118
211
|
const ref = source.ref || DEFAULT_SOURCE_REF;
|
|
119
212
|
const repo = source.repo ?? null;
|
|
120
|
-
//
|
|
121
|
-
//
|
|
122
|
-
//
|
|
213
|
+
// `source_repo` is a LEGACY DISPLAY LABEL that rides into chunk metadata — NOT the
|
|
214
|
+
// ownership decision. Ownership is decided in exactly one place (resolveOwningRepo);
|
|
215
|
+
// this leading-path-segment value is preserved verbatim because rewriting it would
|
|
216
|
+
// re-stamp every existing chunk in every live KB. Do not read it as "which repository
|
|
217
|
+
// owns this file"; for that, ask the owner.
|
|
123
218
|
const source_repo = repo ?? (String(source.path || '').split('/')[0] || 'unknown');
|
|
124
219
|
|
|
125
220
|
if (!repo) {
|
|
221
|
+
// Resolve the commit IN THE REPOSITORY THAT OWNS THE SOURCE — the same one owner the
|
|
222
|
+
// gate consults. Resolving it at the ambient workspace root would record the
|
|
223
|
+
// SUPERREPO's commit for content whose blobs came from a submodule or a sibling repo:
|
|
224
|
+
// two derivations of "where did this content come from" that disagree, which is the
|
|
225
|
+
// mirror-drift this file already refuses elsewhere. A provenance record that names the
|
|
226
|
+
// wrong repository's commit is a FALSE record, not an approximate one.
|
|
227
|
+
const owner = resolveOwningRepo(source.absolutePath);
|
|
228
|
+
if (!owner) {
|
|
229
|
+
throw new Error(
|
|
230
|
+
`Source "${source.path}" resolves to ${source.absolutePath}, which is inside NO git ` +
|
|
231
|
+
`repository. Refusing to record provenance — there is no repository whose ref "${ref}" ` +
|
|
232
|
+
`could describe this content.`
|
|
233
|
+
);
|
|
234
|
+
}
|
|
235
|
+
const provenanceRoot = owner.repoRoot;
|
|
126
236
|
let sha;
|
|
127
237
|
try {
|
|
128
238
|
sha = execSync(`git rev-parse "${ref}^{commit}"`, {
|
|
129
|
-
cwd:
|
|
239
|
+
cwd: provenanceRoot, encoding: 'utf-8', stdio: ['pipe', 'pipe', 'pipe'],
|
|
130
240
|
}).trim();
|
|
131
241
|
} catch {
|
|
132
242
|
throw new Error(
|
|
133
|
-
`Cannot resolve ref "${ref}" to a commit in ${
|
|
134
|
-
`Refusing to record provenance — a sync-state commit of
|
|
135
|
-
`missing one. Check that the ref exists
|
|
243
|
+
`Cannot resolve ref "${ref}" to a commit in ${provenanceRoot} (the repository that owns ` +
|
|
244
|
+
`source "${source.path}") . Refusing to record provenance — a sync-state commit of ` +
|
|
245
|
+
`"unknown" is a FALSE record, not a missing one. Check that the ref exists in THAT ` +
|
|
246
|
+
`repository, which is not necessarily the workspace root ${workspaceRoot}.`
|
|
136
247
|
);
|
|
137
248
|
}
|
|
138
249
|
if (!SHA40.test(sha)) {
|
|
139
250
|
throw new Error(`Ref "${ref}" resolved to "${sha}", which is not a commit sha (source "${source.path}").`);
|
|
140
251
|
}
|
|
141
|
-
|
|
252
|
+
// `repo_root` MAKES THE RECORDED COMMIT SELF-DESCRIBING. Before this, every commit in
|
|
253
|
+
// sync-state was a workspace-root commit, so a reader could resolve it at the workspace
|
|
254
|
+
// root and be right by accident. Now a commit may belong to a submodule or a sibling
|
|
255
|
+
// repo, and the SAME 40 hex characters mean different things depending on which
|
|
256
|
+
// repository you ask. Writing the owning repo beside the commit is what keeps the store
|
|
257
|
+
// homogeneous: a reader resolves the commit WHERE THIS SAYS, and a record written before
|
|
258
|
+
// this field existed is missing it and can be detected rather than silently misread.
|
|
259
|
+
return {
|
|
260
|
+
repo: null, ref, resolved_commit_sha: sha, source_repo,
|
|
261
|
+
repo_root: path.relative(workspaceRoot, provenanceRoot) || '.',
|
|
262
|
+
};
|
|
142
263
|
}
|
|
143
264
|
|
|
144
|
-
// CROSS-REPO: resolve against the remote. Never a local read.
|
|
265
|
+
// CROSS-REPO: resolve against the remote. Never a local read. `repo_root` is null because
|
|
266
|
+
// the content is not in this workspace at all — `repo` already says where it lives.
|
|
145
267
|
if (SHA40.test(ref)) {
|
|
146
|
-
return { repo, ref, resolved_commit_sha: ref.toLowerCase(), source_repo };
|
|
268
|
+
return { repo, ref, resolved_commit_sha: ref.toLowerCase(), source_repo, repo_root: null };
|
|
147
269
|
}
|
|
148
270
|
const remoteUrl = `https://github.com/${repo}.git`;
|
|
149
271
|
let out;
|
|
@@ -167,7 +289,7 @@ export async function resolveSourceProvenance(source, workspaceRoot) {
|
|
|
167
289
|
`Repo "${repo}" has no ref "${ref}" (source "${source.path}"). REFUSING rather than guessing.`
|
|
168
290
|
);
|
|
169
291
|
}
|
|
170
|
-
return { repo, ref, resolved_commit_sha: sha, source_repo };
|
|
292
|
+
return { repo, ref, resolved_commit_sha: sha, source_repo, repo_root: null };
|
|
171
293
|
}
|
|
172
294
|
|
|
173
295
|
/**
|
|
@@ -247,6 +369,9 @@ async function isFile(filePath) {
|
|
|
247
369
|
export async function walkCorpus(manifest, workspaceRoot) {
|
|
248
370
|
const files = [];
|
|
249
371
|
const provenance = [];
|
|
372
|
+
// Every file the source gate refused, collected so ONE error can name them all. A
|
|
373
|
+
// first-throw would hide offenders 2..N and turn one fix into N sync attempts.
|
|
374
|
+
const refused = [];
|
|
250
375
|
let commitSha = 'unknown';
|
|
251
376
|
|
|
252
377
|
// Nested-source dedup (WS-EVP-DESCIX-KB-CLEANUP item 4): a manifest may list both a directory
|
|
@@ -276,6 +401,39 @@ export async function walkCorpus(manifest, workspaceRoot) {
|
|
|
276
401
|
const { absolutePath, ref, tier, doc_type, syncignore } = source;
|
|
277
402
|
const thisSourceRoot = path.resolve(absolutePath);
|
|
278
403
|
|
|
404
|
+
// An IN-REPO source whose path lies in NO repository joins the AGGREGATED refusal rather
|
|
405
|
+
// than throwing. Both outcomes refuse — but throwing would abort on the first offending
|
|
406
|
+
// source and hide every later one, turning one manifest fix into N sync attempts, which
|
|
407
|
+
// is the whole reason this walk aggregates. Measured: `EGPT/` (in no repo) aborted the
|
|
408
|
+
// EVP-EGPT walk before its other sources were ever examined.
|
|
409
|
+
//
|
|
410
|
+
// CROSS-REPO SOURCES ARE DELIBERATELY EXEMPT. Their `absolutePath` is a fiction — the
|
|
411
|
+
// content lives in another repository — so "is it in a repo here" is the wrong question
|
|
412
|
+
// and answering it would MASK the accurate refusal resolveSourceProvenance already owns.
|
|
413
|
+
// Measured: EVP-BEAST's four `repo: eabadir/unk` sources were being reported as "not
|
|
414
|
+
// inside any git repository" instead of "cross-repo content fetch is not implemented".
|
|
415
|
+
if (source.repo === undefined) {
|
|
416
|
+
const exists = await fs.access(absolutePath).then(() => true, () => false);
|
|
417
|
+
if (!exists) {
|
|
418
|
+
refused.push({
|
|
419
|
+
path: source.path, ref, source: source.path,
|
|
420
|
+
reason: `DOES NOT EXIST at ${absolutePath} — the manifest names a path that is not ` +
|
|
421
|
+
`on disk; correct the path or remove this source from the manifest`,
|
|
422
|
+
});
|
|
423
|
+
continue;
|
|
424
|
+
}
|
|
425
|
+
if (!resolveOwningRepo(absolutePath)) {
|
|
426
|
+
refused.push({
|
|
427
|
+
path: source.path, ref, source: source.path,
|
|
428
|
+
reason: `NOT INSIDE ANY GIT REPOSITORY (${absolutePath}) — no repository owns this ` +
|
|
429
|
+
`path, so ref "${ref}" cannot be resolved for it and nothing can vouch for ` +
|
|
430
|
+
`its content; remove this source from the manifest, or put the content in a ` +
|
|
431
|
+
`repository and commit it`,
|
|
432
|
+
});
|
|
433
|
+
continue;
|
|
434
|
+
}
|
|
435
|
+
}
|
|
436
|
+
|
|
279
437
|
// Resolve this source's exact provenance BEFORE reading anything. A cross-repo source that
|
|
280
438
|
// cannot be fetched REFUSES here, naming the repo — it is never silently skipped and never
|
|
281
439
|
// falls back to reading an adjacent working tree.
|
|
@@ -318,13 +476,17 @@ export async function walkCorpus(manifest, workspaceRoot) {
|
|
|
318
476
|
if (ownerSourceRoot(filePath) !== thisSourceRoot) {
|
|
319
477
|
continue;
|
|
320
478
|
}
|
|
321
|
-
//
|
|
479
|
+
// Workspace-relative path is the DISPLAY/metadata spelling only. Git resolution uses
|
|
480
|
+
// the owning repository — see resolveTrackedBlobSha.
|
|
322
481
|
const workspaceRelativePath = path.relative(workspaceRoot, filePath);
|
|
323
482
|
|
|
324
|
-
//
|
|
325
|
-
|
|
483
|
+
// THE SOURCE GATE. A file the OWNING repository does not track at this source's ref
|
|
484
|
+
// is REFUSED by name — never silently chunked from the working tree, and never
|
|
485
|
+
// silently skipped either (a skip on a clean worktree would PURGE its live chunks).
|
|
486
|
+
const { sha: blobSha, reason } = resolveTrackedBlobSha(filePath, ref, workspaceRoot);
|
|
326
487
|
if (!blobSha) {
|
|
327
|
-
|
|
488
|
+
refused.push({ path: workspaceRelativePath, ref, reason, source: source.path });
|
|
489
|
+
continue;
|
|
328
490
|
}
|
|
329
491
|
|
|
330
492
|
// "Which repo does this content belong to" has ONE owner: resolveSourceProvenance.
|
|
@@ -351,7 +513,36 @@ export async function walkCorpus(manifest, workspaceRoot) {
|
|
|
351
513
|
}
|
|
352
514
|
}
|
|
353
515
|
|
|
354
|
-
|
|
516
|
+
// ONE aggregated refusal naming every offender, its ref and its reason. Raised AFTER
|
|
517
|
+
// the full walk so the operator fixes the manifest once rather than N times, and BEFORE
|
|
518
|
+
// any caller can chunk, upsert or purge — the refused files are neither served nor
|
|
519
|
+
// treated as deletions.
|
|
520
|
+
if (refused.length > 0) {
|
|
521
|
+
// The error must be fixable IN ONE ACT from its text alone: it names the manifest
|
|
522
|
+
// FILE to edit and the offending ENTRY within it, not merely the resolved path.
|
|
523
|
+
// `_manifestPath` is owned by ManifestLoader — consume it, never re-derive it.
|
|
524
|
+
const manifestFile = manifest._manifestPath || `(manifest for kb "${manifest.kb_name}")`;
|
|
525
|
+
const lines = refused.map(
|
|
526
|
+
r => ` - ${r.path}\n ref: "${r.ref}" manifest entry: sources[] path "${r.source}"\n ${r.reason}`
|
|
527
|
+
);
|
|
528
|
+
const err = new Error(
|
|
529
|
+
`Corpus source gate FAILED for KB "${manifest.kb_name}": ${refused.length} file(s) are ` +
|
|
530
|
+
`not tracked by git at their manifest ref — refusing to sync.\n` +
|
|
531
|
+
`Manifest to edit: ${manifestFile}\n${lines.join('\n')}\n` +
|
|
532
|
+
`Every corpus source must be committed at its ref. There is no working-tree fallback: ` +
|
|
533
|
+
`content git excludes must never be served as a citable KB source.`
|
|
534
|
+
);
|
|
535
|
+
// The offenders as DATA, so a reporting tool consumes the gate's own verdict instead of
|
|
536
|
+
// re-implementing the predicate or scraping this prose. The message stays the human
|
|
537
|
+
// surface; `refused` is the machine one, and they cannot disagree because there is one
|
|
538
|
+
// decision behind both.
|
|
539
|
+
err.refused = refused;
|
|
540
|
+
err.kb_name = manifest.kb_name;
|
|
541
|
+
err.manifestPath = manifestFile;
|
|
542
|
+
throw err;
|
|
543
|
+
}
|
|
544
|
+
|
|
545
|
+
return { files, commitSha, provenance, gateBoundary: SOURCE_GATE_BOUNDARY };
|
|
355
546
|
}
|
|
356
547
|
|
|
357
|
-
export default { walkCorpus, resolveSourceProvenance };
|
|
548
|
+
export default { walkCorpus, resolveSourceProvenance, resolveTrackedBlobSha, SOURCE_GATE_BOUNDARY };
|
|
@@ -13,6 +13,7 @@
|
|
|
13
13
|
*/
|
|
14
14
|
|
|
15
15
|
import * as fs from 'fs/promises';
|
|
16
|
+
import { statSync, realpathSync } from 'fs';
|
|
16
17
|
import * as path from 'path';
|
|
17
18
|
import { execFileSync } from 'child_process';
|
|
18
19
|
// Canonical doc_class taxonomy lives in CorpusDenyLint — import it, never re-list.
|
|
@@ -110,6 +111,73 @@ export function detectRepoSlug(dir) {
|
|
|
110
111
|
}
|
|
111
112
|
}
|
|
112
113
|
|
|
114
|
+
/**
|
|
115
|
+
* THE ONE OWNER OF "WHICH REPOSITORY OWNS THIS FILE".
|
|
116
|
+
*
|
|
117
|
+
* A workspace is NOT one repository. This one crosses three kinds of boundary at once:
|
|
118
|
+
* the superrepo itself; git SUBMODULES (DeSciX/DeSciX_Cloud|Core|Powch are gitlinks —
|
|
119
|
+
* `git ls-tree HEAD DeSciX/` shows mode 160000); and SIBLING repos that are not submodules
|
|
120
|
+
* at all but independent checkouts the superrepo deliberately ignores (EGPT-research via
|
|
121
|
+
* .gitignore:9, FRAQTL — each with its own `.git` and its own `origin` remote).
|
|
122
|
+
*
|
|
123
|
+
* A REF HAS NO MEANING EXCEPT RELATIVE TO A REPOSITORY. Resolving one against the ambient
|
|
124
|
+
* workspace root is what made this gate refuse 596 files that ARE committed: the superrepo
|
|
125
|
+
* sees a submodule only as a gitlink commit, and an ignored sibling repo not at all.
|
|
126
|
+
* `git rev-parse --show-toplevel` walks UP from the path and stops at the FIRST enclosing
|
|
127
|
+
* repository — which is precisely the repository whose branches the source's `ref` names,
|
|
128
|
+
* and which settles submodule and sibling boundaries with ONE mechanism rather than two.
|
|
129
|
+
*
|
|
130
|
+
* This is NOT the question `detectRepoSlug` answers. That one is "what is this repository
|
|
131
|
+
* CALLED" (for the cross-repo self-naming refusal); this one is "where does it BEGIN", so a
|
|
132
|
+
* ref and a repo-relative path can be formed. Both live here so repository identity has a
|
|
133
|
+
* single home and no consumer re-derives either.
|
|
134
|
+
*
|
|
135
|
+
* @param {string} absolutePath - absolute path to a file or directory in the workspace
|
|
136
|
+
* @returns {{repoRoot: string, repoRelativePath: string}|null} null when the path lies in
|
|
137
|
+
* NO git repository at all — callers must fail loud, never assume a root.
|
|
138
|
+
*/
|
|
139
|
+
const OWNING_REPO_CACHE = new Map(); // real directory -> repoRoot. POSITIVE results only.
|
|
140
|
+
export function resolveOwningRepo(absolutePath) {
|
|
141
|
+
const abs = path.resolve(absolutePath);
|
|
142
|
+
// `--show-toplevel` needs a directory that exists, and a source may name a file. Split
|
|
143
|
+
// off the basename ONLY for a non-directory: for a DIRECTORY source, `path.dirname` would
|
|
144
|
+
// step over the very boundary being resolved (dirname of `DeSciX/DeSciX_Cloud` is the
|
|
145
|
+
// superrepo, which is the wrong answer).
|
|
146
|
+
let dir = abs, base = null;
|
|
147
|
+
try {
|
|
148
|
+
if (!statSync(abs).isDirectory()) { dir = path.dirname(abs); base = path.basename(abs); }
|
|
149
|
+
} catch {
|
|
150
|
+
dir = path.dirname(abs); base = path.basename(abs);
|
|
151
|
+
}
|
|
152
|
+
|
|
153
|
+
// BOTH SIDES MUST BE REAL PATHS OR THE SUBTRACTION IS GARBAGE. `git rev-parse
|
|
154
|
+
// --show-toplevel` always answers with symlinks resolved, so comparing it against a
|
|
155
|
+
// caller path that still contains one yields a `../../..`-riddled relative path that
|
|
156
|
+
// matches nothing. This is not a corner case: on macOS every temp dir is /var ->
|
|
157
|
+
// /private/var, and this repo's own dev flow serves the Core tree through a symlink.
|
|
158
|
+
let realDir;
|
|
159
|
+
try { realDir = realpathSync(dir); } catch { realDir = dir; }
|
|
160
|
+
|
|
161
|
+
let repoRoot = OWNING_REPO_CACHE.get(realDir);
|
|
162
|
+
if (repoRoot === undefined) {
|
|
163
|
+
try {
|
|
164
|
+
repoRoot = execFileSync('git', ['rev-parse', '--show-toplevel'], {
|
|
165
|
+
cwd: realDir, encoding: 'utf-8', stdio: ['pipe', 'pipe', 'pipe'],
|
|
166
|
+
}).trim();
|
|
167
|
+
} catch {
|
|
168
|
+
return null; // No repository. NOT cached: a negative is not durable — the directory
|
|
169
|
+
} // may be `git init`-ed later in the same process (fixtures do exactly this).
|
|
170
|
+
if (!repoRoot) return null;
|
|
171
|
+
OWNING_REPO_CACHE.set(realDir, repoRoot);
|
|
172
|
+
}
|
|
173
|
+
|
|
174
|
+
const repoRelativeDir = path.relative(repoRoot, realDir);
|
|
175
|
+
return {
|
|
176
|
+
repoRoot,
|
|
177
|
+
repoRelativePath: base ? path.join(repoRelativeDir, base) : repoRelativeDir,
|
|
178
|
+
};
|
|
179
|
+
}
|
|
180
|
+
|
|
113
181
|
/**
|
|
114
182
|
* Enforce the two-shape source contract on ONE source entry.
|
|
115
183
|
*
|