@descix/cli 1.0.6 → 1.0.8
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +1 -1
- package/agent-assets/guides/README.md +1 -1
- package/agent-assets/guides/dynamic-site-pattern.md +17 -30
- package/agent-assets/guides/knowledge-only-pattern.md +11 -15
- package/agent-assets/guides/microservice-pattern.md +12 -14
- package/agent-assets/guides/static-site-pattern.md +12 -15
- package/agent-assets/instructions/app-asset-validation.md +1 -1
- package/agent-assets/instructions/onboarding-flow.md +4 -4
- package/agent-assets/reference/app-creation.md +36 -101
- package/agent-assets/reference/architecture.md +43 -51
- package/agent-assets/reference/local-dev.md +1 -6
- package/agent-assets/reference/mcp-integration.md +21 -23
- package/agent-assets/reference/workspace-config.md +97 -340
- package/bin/descix.js +337 -618
- package/lib/api-client.js +30 -22
- package/lib/command-visibility.js +213 -0
- package/lib/commands/airdrop.js +10 -0
- package/lib/commands/auth.js +16 -2
- package/lib/commands/config.js +13 -31
- package/lib/commands/corpus.js +163 -59
- package/lib/commands/health.js +116 -21
- package/lib/commands/init.js +17 -45
- package/lib/commands/kb.js +7 -0
- package/lib/commands/retired-kb-sync.js +5 -27
- package/lib/commands/status.js +0 -9
- package/lib/core/Chunker.js +1 -1
- package/lib/core/CorpusWalker.js +238 -47
- package/lib/core/ManifestLoader.js +68 -0
- package/lib/core/Syncer.js +147 -46
- package/lib/interactive.js +158 -0
- package/lib/origin.js +3 -3
- package/lib/sdk-assets.js +28 -0
- package/lib/verb-invokes.generated.json +233 -0
- package/lib/wallet-file.js +36 -37
- package/lib/workspace-config.js +266 -199
- package/lib/workspace-identity.js +0 -1
- package/package.json +7 -3
- package/templates/agent-claude.md +1 -3
- package/templates/agent-cline.md +1 -2
- package/templates/agent-copilot.md +1 -2
- package/templates/agent-cursor.md +1 -2
- package/templates/scaffolds/microservice/SERVICE_README_sdk.md +1 -1
- package/templates/scaffolds/microservice/templates/SERVICE_README_TEMPLATE.md +2 -3
- package/agent-assets/rules/descix_mcp.template.mdc +0 -243
- package/lib/commands/briefer/errors.js +0 -88
- package/lib/commands/briefer/index.js +0 -319
- package/lib/commands/briefer/sources/canonical-sources.js +0 -129
- package/lib/commands/briefer/sources/entitlements.js +0 -220
- package/lib/commands/briefer/sources/environments.js +0 -205
- package/lib/commands/briefer/sources/identifiers.js +0 -214
- package/lib/commands/briefer/sources/microservice-deploy.js +0 -267
- package/lib/commands/briefer/sources/routing.js +0 -275
- package/lib/commands/briefer/sources/what-is-not.js +0 -223
- package/lib/commands/briefer/stitcher.js +0 -134
- package/lib/commands/briefer/util/source-reader.js +0 -502
- package/lib/commands/mcp.js +0 -273
- package/lib/commands/update.js +0 -581
- package/lib/wizard/setup.js +0 -503
- package/templates/descix_mcp.mdc +0 -122
package/lib/commands/init.js
CHANGED
|
@@ -5,7 +5,6 @@
|
|
|
5
5
|
*/
|
|
6
6
|
|
|
7
7
|
import chalk from 'chalk';
|
|
8
|
-
import readline from 'readline';
|
|
9
8
|
import path from 'path';
|
|
10
9
|
import fs from 'fs/promises';
|
|
11
10
|
import { fileURLToPath } from 'url';
|
|
@@ -17,56 +16,29 @@ import { generateAgentFiles } from '../agent-files.js';
|
|
|
17
16
|
// spelled here is a second derivation of the same fact, and the removed verb this replaced
|
|
18
17
|
// reached a developer who had done everything right.
|
|
19
18
|
import { CANONICAL_KB_SYNC } from './retired-kb-sync.js';
|
|
19
|
+
// "May I prompt?" has ONE OWNER. init does not derive it, and holds no TTY check of its own.
|
|
20
|
+
import { createPromptSession } from '../interactive.js';
|
|
20
21
|
|
|
21
22
|
const __filename = fileURLToPath(import.meta.url);
|
|
22
23
|
const __dirname = dirname(__filename);
|
|
23
24
|
|
|
24
|
-
/**
|
|
25
|
-
* Create readline interface for interactive prompts
|
|
26
|
-
*/
|
|
27
|
-
function createPrompt() {
|
|
28
|
-
return readline.createInterface({
|
|
29
|
-
input: process.stdin,
|
|
30
|
-
output: process.stdout
|
|
31
|
-
});
|
|
32
|
-
}
|
|
33
|
-
|
|
34
|
-
/**
|
|
35
|
-
* Ask for text input with optional default
|
|
36
|
-
*/
|
|
37
|
-
function askInput(rl, question, defaultValue = '') {
|
|
38
|
-
return new Promise((resolve) => {
|
|
39
|
-
const prompt = defaultValue ? `${question} [${defaultValue}]: ` : `${question}: `;
|
|
40
|
-
rl.question(prompt, (answer) => {
|
|
41
|
-
resolve(answer.trim() || defaultValue);
|
|
42
|
-
});
|
|
43
|
-
});
|
|
44
|
-
}
|
|
45
|
-
|
|
46
|
-
/**
|
|
47
|
-
* Ask a yes/no question
|
|
48
|
-
*/
|
|
49
|
-
function askYesNo(rl, question, defaultYes = true) {
|
|
50
|
-
return new Promise((resolve) => {
|
|
51
|
-
const suffix = defaultYes ? '(Y/n)' : '(y/N)';
|
|
52
|
-
rl.question(`${question} ${suffix}: `, (answer) => {
|
|
53
|
-
const trimmed = answer.trim().toLowerCase();
|
|
54
|
-
if (trimmed === '') {
|
|
55
|
-
resolve(defaultYes);
|
|
56
|
-
} else {
|
|
57
|
-
resolve(trimmed === 'y' || trimmed === 'yes');
|
|
58
|
-
}
|
|
59
|
-
});
|
|
60
|
-
});
|
|
61
|
-
}
|
|
62
|
-
|
|
63
25
|
/**
|
|
64
26
|
* Run the init command
|
|
65
27
|
* @param {Object} apiClient - API client (optional, for listing communities)
|
|
66
28
|
* @param {Object} options - Command options
|
|
67
29
|
*/
|
|
68
30
|
export async function runInit(apiClient, options = {}) {
|
|
69
|
-
|
|
31
|
+
// The session is GATED AT CONSTRUCTION: under a non-TTY this throws NonInteractiveError,
|
|
32
|
+
// which bin/descix.js's init action prints and exits 1 on. It can no longer hang on an open
|
|
33
|
+
// pipe/FIFO, nor exit 0 writing nothing on /dev/null.
|
|
34
|
+
const rl = createPromptSession({
|
|
35
|
+
what: 'descix init',
|
|
36
|
+
nonInteractiveForm: [
|
|
37
|
+
'descix init has no non-interactive form today. Run it in a terminal.',
|
|
38
|
+
'',
|
|
39
|
+
'Nothing has been changed. An existing .descix/workspace.json is left as it is.'
|
|
40
|
+
]
|
|
41
|
+
});
|
|
70
42
|
const projectPath = options.path ? path.resolve(options.path) : process.cwd();
|
|
71
43
|
|
|
72
44
|
// If --from-invite provided, resolve the invite first to pre-fill context
|
|
@@ -134,7 +106,7 @@ export async function runInit(apiClient, options = {}) {
|
|
|
134
106
|
}
|
|
135
107
|
|
|
136
108
|
if (hasExisting && !options.force) {
|
|
137
|
-
const overwrite = await askYesNo(
|
|
109
|
+
const overwrite = await rl.askYesNo(chalk.yellow('Workspace already initialized. Overwrite?'), false);
|
|
138
110
|
if (!overwrite) {
|
|
139
111
|
console.log(chalk.gray('\nInitialization cancelled.\n'));
|
|
140
112
|
rl.close();
|
|
@@ -160,7 +132,7 @@ export async function runInit(apiClient, options = {}) {
|
|
|
160
132
|
// ignore
|
|
161
133
|
}
|
|
162
134
|
}
|
|
163
|
-
communityId = await
|
|
135
|
+
communityId = await rl.ask(chalk.white('Community ID'));
|
|
164
136
|
if (!communityId) {
|
|
165
137
|
console.log(chalk.red('\n❌ Community ID is required.\n'));
|
|
166
138
|
rl.close();
|
|
@@ -171,7 +143,7 @@ export async function runInit(apiClient, options = {}) {
|
|
|
171
143
|
const defaultAppName = path.basename(projectPath).toLowerCase().replace(/[^a-z0-9]/g, '_');
|
|
172
144
|
let appName = options.app;
|
|
173
145
|
if (!appName) {
|
|
174
|
-
appName = await
|
|
146
|
+
appName = await rl.ask(chalk.white('App name'), defaultAppName);
|
|
175
147
|
if (!appName) {
|
|
176
148
|
console.log(chalk.red('\n❌ App name is required.\n'));
|
|
177
149
|
rl.close();
|
|
@@ -185,7 +157,7 @@ export async function runInit(apiClient, options = {}) {
|
|
|
185
157
|
console.log(chalk.white(` Community: ${communityId}`));
|
|
186
158
|
console.log(chalk.white(` App: ${appName} (${appId})`));
|
|
187
159
|
|
|
188
|
-
const proceed = await askYesNo(
|
|
160
|
+
const proceed = await rl.askYesNo(chalk.white('\nProceed?'), true);
|
|
189
161
|
if (!proceed) {
|
|
190
162
|
console.log(chalk.gray('\nCancelled.\n'));
|
|
191
163
|
rl.close();
|
package/lib/commands/kb.js
CHANGED
|
@@ -22,6 +22,8 @@ import ora from 'ora';
|
|
|
22
22
|
import * as path from 'path';
|
|
23
23
|
import * as fs from 'fs/promises';
|
|
24
24
|
import inquirer from 'inquirer';
|
|
25
|
+
// "May I prompt?" has ONE OWNER. --interactive is an opt-IN, not a guarantee a human is there.
|
|
26
|
+
import { requireInteractive } from '../interactive.js';
|
|
25
27
|
import { WorkspaceConfig } from '../workspace-config.js';
|
|
26
28
|
import { hydrateKb, pushStaging, checkStagingFiles } from '../core/Hydrator.js';
|
|
27
29
|
import { processKb } from '../core/Chunker.js';
|
|
@@ -176,6 +178,11 @@ export async function runKbPush(apiClient, options) {
|
|
|
176
178
|
// Create conflict prompt callback for interactive mode
|
|
177
179
|
const onConflictPrompt = options.interactive ? async (fileName, fileInfo) => {
|
|
178
180
|
spinner.stop();
|
|
181
|
+
requireInteractive({
|
|
182
|
+
what: 'descix drive push --interactive',
|
|
183
|
+
question: `File "${fileName}" exists in Drive. Overwrite or skip?`,
|
|
184
|
+
nonInteractiveForm: ['descix drive push # without --interactive, conflicts resolve without a prompt']
|
|
185
|
+
});
|
|
179
186
|
const { action } = await inquirer.prompt([{
|
|
180
187
|
type: 'list',
|
|
181
188
|
name: 'action',
|
|
@@ -19,20 +19,17 @@
|
|
|
19
19
|
export const CANONICAL_KB_SYNC = 'descix kb corpus sync';
|
|
20
20
|
|
|
21
21
|
/** The verb that creates a KB — `kb corpus sync`'s dependency (I4). */
|
|
22
|
-
export const CANONICAL_KB_CREATE = 'descix kb
|
|
22
|
+
export const CANONICAL_KB_CREATE = 'descix app init --kb <kb_name>';
|
|
23
23
|
|
|
24
24
|
/**
|
|
25
25
|
* Every KB-sync surface retired by ws-devplane-cli-kb-surface-single-canonical-sync.
|
|
26
26
|
* `invocation` is what a user types; `parent` is the commander parent it hung off.
|
|
27
27
|
*/
|
|
28
28
|
export const RETIRED_KB_SYNC_SURFACES = [
|
|
29
|
-
{ id: 'sync', parent: 'program', name: 'sync', invocation: 'descix sync'
|
|
30
|
-
{ id: 'sync.kb', parent: 'sync', name: 'kb', invocation: 'descix sync kb'
|
|
31
|
-
{ id: 'kb.chunk', parent: 'kb', name: 'chunk', invocation: 'descix kb chunk'
|
|
32
|
-
{ id: 'kb.sync', parent: 'kb', name: 'sync', invocation: 'descix kb sync'
|
|
33
|
-
// registered:false -- `update` survives for app/site, so this one is a dispatcher branch
|
|
34
|
-
// inside the live `update` command rather than a commander registration of its own.
|
|
35
|
-
{ id: 'update.kb', parent: 'update', name: 'kb', invocation: 'descix update kb', registered: false },
|
|
29
|
+
{ id: 'sync', parent: 'program', name: 'sync', invocation: 'descix sync' },
|
|
30
|
+
{ id: 'sync.kb', parent: 'sync', name: 'kb', invocation: 'descix sync kb' },
|
|
31
|
+
{ id: 'kb.chunk', parent: 'kb', name: 'chunk', invocation: 'descix kb chunk' },
|
|
32
|
+
{ id: 'kb.sync', parent: 'kb', name: 'sync', invocation: 'descix kb sync' },
|
|
36
33
|
];
|
|
37
34
|
|
|
38
35
|
/** Implementations deleted with those surfaces. No exported symbol survives without a caller (I3). */
|
|
@@ -61,24 +58,6 @@ export function retiredKbSyncMessage(invocation) {
|
|
|
61
58
|
);
|
|
62
59
|
}
|
|
63
60
|
|
|
64
|
-
/**
|
|
65
|
-
* Refuse a retired invocation LOUDLY and exit non-zero.
|
|
66
|
-
*
|
|
67
|
-
* MEASURED, not assumed: a `throw` from updateAuto is NOT swallowed — updateAuto's own catch
|
|
68
|
-
* prints the message before rethrowing, so both forms reach the user and exit 1. This exists
|
|
69
|
-
* for two smaller, real reasons: (1) ONE owner for the refusal, so the dispatcher branch and
|
|
70
|
-
* updateAuto's kb branch cannot drift into different wording or different exit codes; and
|
|
71
|
-
* (2) a deliberate retirement is not a crash, so it should not be framed as "✖ Update failed".
|
|
72
|
-
*
|
|
73
|
-
* @param {string} invocation - the retired invocation, e.g. 'descix update kb'
|
|
74
|
-
* @param {(s:string)=>string} red - chalk.red or equivalent
|
|
75
|
-
* @returns {never}
|
|
76
|
-
*/
|
|
77
|
-
export function refuseRetiredKbSync(invocation, red) {
|
|
78
|
-
console.error(red(`\n❌ ${retiredKbSyncMessage(invocation)}\n`));
|
|
79
|
-
process.exit(1);
|
|
80
|
-
}
|
|
81
|
-
|
|
82
61
|
/**
|
|
83
62
|
* Register a retired invocation so it exits NON-ZERO naming the canonical surface.
|
|
84
63
|
* Accepts and ignores any args/options the old surface took, so an old script fails with
|
|
@@ -120,7 +99,6 @@ export function registerRetiredKbSync(parent, name, invocation, red) {
|
|
|
120
99
|
export function registerAllRetiredKbSync(roots, red) {
|
|
121
100
|
const byId = { ...roots };
|
|
122
101
|
for (const s of RETIRED_KB_SYNC_SURFACES) {
|
|
123
|
-
if (!s.registered) continue;
|
|
124
102
|
const parent = byId[s.parent];
|
|
125
103
|
if (!parent) {
|
|
126
104
|
throw new Error(
|
package/lib/commands/status.js
CHANGED
|
@@ -206,15 +206,6 @@ export async function runStatus(options = {}) {
|
|
|
206
206
|
console.log(`Service Key: ${chalk.gray('None')} (dev-overrides.json)`);
|
|
207
207
|
}
|
|
208
208
|
|
|
209
|
-
// Check for MCP rules
|
|
210
|
-
const rulesPath = path.join(checkDir, '.cursor', 'rules', 'descix_mcp.mdc');
|
|
211
|
-
try {
|
|
212
|
-
await fs.access(rulesPath);
|
|
213
|
-
console.log(`MCP Rules: ${chalk.green('Deployed')} (.cursor/rules/descix_mcp.mdc)`);
|
|
214
|
-
} catch {
|
|
215
|
-
console.log(`MCP Rules: ${chalk.gray('Not Deployed')}`);
|
|
216
|
-
}
|
|
217
|
-
|
|
218
209
|
// Check gcloud
|
|
219
210
|
const gcloudStatus = await checkGcloud();
|
|
220
211
|
if (gcloudStatus.installed) {
|
package/lib/core/Chunker.js
CHANGED
|
@@ -938,7 +938,7 @@ export async function processKb(config, options = {}) {
|
|
|
938
938
|
try {
|
|
939
939
|
await fs.access(srcDir);
|
|
940
940
|
} catch {
|
|
941
|
-
throw new Error(`Source directory not found: ${srcDir}\nRun "descix
|
|
941
|
+
throw new Error(`Source directory not found: ${srcDir}\nRun "descix drive pull" first.`);
|
|
942
942
|
}
|
|
943
943
|
|
|
944
944
|
await fs.mkdir(chunksDir, { recursive: true });
|
package/lib/core/CorpusWalker.js
CHANGED
|
@@ -7,16 +7,19 @@
|
|
|
7
7
|
* Architecture:
|
|
8
8
|
* - Files stay in place — no copying or staging
|
|
9
9
|
* - Git blob SHAs provide versioning (same content = same SHA)
|
|
10
|
-
* -
|
|
10
|
+
* - GIT IS THE ONLY SOURCE OF TRUTH FOR WHAT IS A CORPUS SOURCE: a file must be TRACKED
|
|
11
|
+
* AT THE MANIFEST'S REF or it is refused by name. There is no working-tree fallback.
|
|
11
12
|
* - Chunk IDs are content-addressed: {app_id}:{kb_name}:{blob_sha}:{chunk_idx}
|
|
12
13
|
*/
|
|
13
14
|
|
|
14
15
|
import * as fs from 'fs/promises';
|
|
16
|
+
import * as fsSync from 'fs';
|
|
15
17
|
import * as path from 'path';
|
|
16
18
|
import { execSync } from 'child_process';
|
|
17
19
|
import { minimatch } from 'minimatch';
|
|
18
|
-
// The source-ref default
|
|
19
|
-
|
|
20
|
+
// The source-ref default and "which repository owns this file" are both owned by
|
|
21
|
+
// ManifestLoader — consume them, never re-default and never re-derive.
|
|
22
|
+
import { DEFAULT_SOURCE_REF, resolveOwningRepo } from './ManifestLoader.js';
|
|
20
23
|
|
|
21
24
|
/**
|
|
22
25
|
* File extensions that are processable for RAG chunking.
|
|
@@ -60,42 +63,132 @@ function isIgnored(relativePath, syncignore) {
|
|
|
60
63
|
return false;
|
|
61
64
|
}
|
|
62
65
|
|
|
66
|
+
const SHA40 = /^[0-9a-f]{40}$/i;
|
|
67
|
+
|
|
63
68
|
/**
|
|
64
|
-
*
|
|
65
|
-
*
|
|
66
|
-
*
|
|
69
|
+
* Explain WHICH WAY a path failed the "tracked at ref" predicate. Purely diagnostic —
|
|
70
|
+
* it never decides anything, so there is still exactly one decision site. Each branch
|
|
71
|
+
* ends in an instruction the operator can act on without further investigation.
|
|
67
72
|
*
|
|
68
|
-
* @
|
|
69
|
-
* @param {string} workspaceRelativePath - Path relative to workspace root
|
|
70
|
-
* @param {string} ref - Git ref (branch/tag)
|
|
71
|
-
* @param {string} workspaceRoot - Workspace root directory
|
|
72
|
-
* @returns {string} Git blob SHA (40 hex chars)
|
|
73
|
+
* @returns {string} the reason string embedded in the aggregated refusal
|
|
73
74
|
*/
|
|
74
|
-
function
|
|
75
|
+
function diagnoseUntracked(repoRelativePath, ref, repoRoot, repoLabel) {
|
|
76
|
+
// Every diagnostic below runs IN THE OWNING REPOSITORY. Running them at the ambient
|
|
77
|
+
// workspace root is what produced the maximally wrong advice this gate was blocked for
|
|
78
|
+
// ("UNTRACKED — commit it on main" about content that was already committed, and
|
|
79
|
+
// "IGNORED BY GIT" about a sibling repo the SUPERREPO ignores but which tracks it fine).
|
|
80
|
+
const where = `in ${repoLabel}`;
|
|
81
|
+
|
|
82
|
+
// (c) Present at HEAD but absent at THIS ref — checked first, because .gitignore
|
|
83
|
+
// patterns do not apply to tracked files and would misdiagnose this case.
|
|
75
84
|
try {
|
|
76
|
-
|
|
77
|
-
|
|
78
|
-
|
|
79
|
-
|
|
85
|
+
execSync(`git rev-parse "HEAD:${repoRelativePath}"`,
|
|
86
|
+
{ cwd: repoRoot, encoding: 'utf-8', stdio: ['pipe', 'pipe', 'pipe'] });
|
|
87
|
+
return `present at HEAD but ABSENT AT REF "${ref}" ${where} — the manifest pins a ref ` +
|
|
88
|
+
`that predates this file; commit it on "${ref}" or point the source at the right ref`;
|
|
89
|
+
} catch { /* not at HEAD either — fall through */ }
|
|
90
|
+
|
|
91
|
+
// (a) Deliberately excluded from the repository. The measured defect class.
|
|
92
|
+
try {
|
|
93
|
+
const hit = execSync(`git check-ignore -v "${repoRelativePath}"`,
|
|
94
|
+
{ cwd: repoRoot, encoding: 'utf-8', stdio: ['pipe', 'pipe', 'pipe'] }
|
|
95
|
+
).trim().split('\n')[0];
|
|
96
|
+
if (hit) {
|
|
97
|
+
return `IGNORED BY GIT (${hit.split('\t')[0]}) ${where} and not tracked at ref ` +
|
|
98
|
+
`"${ref}" — content the repository deliberately excludes must never become a ` +
|
|
99
|
+
`citable KB source; remove this source from the manifest`;
|
|
100
|
+
}
|
|
101
|
+
} catch { /* not ignored — fall through */ }
|
|
102
|
+
|
|
103
|
+
// (b) Simply never committed.
|
|
104
|
+
return `UNTRACKED at ref "${ref}" ${where} — the file exists on disk but that repository ` +
|
|
105
|
+
`does not track it; commit it on "${ref}", or remove this source from the manifest`;
|
|
106
|
+
}
|
|
107
|
+
|
|
108
|
+
/**
|
|
109
|
+
* THE SOURCE GATE — the single owner of "is this file a legitimate corpus source".
|
|
110
|
+
*
|
|
111
|
+
* A source is legitimate iff GIT TRACKS IT AT THE MANIFEST'S REF. That one predicate is
|
|
112
|
+
* the whole rule, and it is deliberately not split by WHY a path fails it: gitignored
|
|
113
|
+
* scratch, never-committed, and absent-at-an-older-ref are all "not tracked at ref".
|
|
114
|
+
* Splitting them would re-derive "what is a corpus source" in a second place, and the
|
|
115
|
+
* copies would drift.
|
|
116
|
+
*
|
|
117
|
+
* THE WORKING TREE IS NEVER CONSULTED. Reading a sha from disk when the ref lookup failed
|
|
118
|
+
* is the measured defect: it served content git was explicitly told to exclude as a
|
|
119
|
+
* citable KB source (corpus:04406066... = `.claude/agent-memory/evp-descix/qa-pairs.jsonl`,
|
|
120
|
+
* excluded by .gitignore, chunked and served anyway). That path is deleted, not fenced
|
|
121
|
+
* behind a flag, and there is nothing that restores it — SUPER-DRY, CEO-D-2026-07-26.
|
|
122
|
+
*
|
|
123
|
+
* THE REF IS RESOLVED IN THE REPOSITORY THAT OWNS THE FILE, never in an ambient root. A
|
|
124
|
+
* workspace spans several repositories (superrepo, submodules, ignored sibling checkouts)
|
|
125
|
+
* and a ref names a branch in exactly one of them. Asking the superrepo about a file inside
|
|
126
|
+
* a submodule asks the wrong repository a question it cannot answer — it holds only a
|
|
127
|
+
* gitlink commit — and the honest-looking "not tracked" that comes back is FALSE. That was
|
|
128
|
+
* measured on this very branch: 596 committed files refused, the two largest live KBs
|
|
129
|
+
* stopped, with advice telling the operator to commit content already committed.
|
|
130
|
+
* "Which repository owns this file" has ONE owner: ManifestLoader::resolveOwningRepo.
|
|
131
|
+
*
|
|
132
|
+
* Returns a REASON rather than throwing so the caller can aggregate every offender into
|
|
133
|
+
* one refusal instead of failing on the first (mirroring CorpusDenyLint's report/assert
|
|
134
|
+
* split — the established refusal shape in this codebase).
|
|
135
|
+
*
|
|
136
|
+
* @param {string} absoluteFilePath - Absolute path to the file under test
|
|
137
|
+
* @param {string} ref - Git ref (branch/tag) the manifest source named
|
|
138
|
+
* @param {string} [workspaceRoot] - Only to label paths in messages; NEVER a resolution root
|
|
139
|
+
* @returns {{sha: string|null, reason: string|null, repoRoot: string|null}}
|
|
140
|
+
*/
|
|
141
|
+
export function resolveTrackedBlobSha(absoluteFilePath, ref, workspaceRoot = null) {
|
|
142
|
+
const owner = resolveOwningRepo(absoluteFilePath);
|
|
143
|
+
if (!owner) {
|
|
144
|
+
return {
|
|
145
|
+
sha: null, repoRoot: null,
|
|
146
|
+
reason: `NOT INSIDE ANY GIT REPOSITORY — no repository owns this path, so ref "${ref}" ` +
|
|
147
|
+
`cannot be resolved for it and nothing can vouch for its content; remove this ` +
|
|
148
|
+
`source from the manifest, or put the content in a repository and commit it`
|
|
149
|
+
};
|
|
150
|
+
}
|
|
151
|
+
const { repoRoot, repoRelativePath } = owner;
|
|
152
|
+
// How the owning repo is named in messages: relative to the workspace when we have one,
|
|
153
|
+
// so the operator reads "DeSciX/DeSciX_Cloud" rather than an absolute machine path.
|
|
154
|
+
// `repoRoot` is a REAL path (git resolves symlinks), so the workspace root must be one
|
|
155
|
+
// too or the subtraction escapes upward into `../../..`.
|
|
156
|
+
let workspaceReal = workspaceRoot;
|
|
157
|
+
try { if (workspaceRoot) workspaceReal = fsSync.realpathSync(workspaceRoot); } catch { /* keep as given */ }
|
|
158
|
+
const repoLabel = workspaceRoot ? (path.relative(workspaceReal, repoRoot) || '.') : repoRoot;
|
|
159
|
+
|
|
160
|
+
let sha;
|
|
161
|
+
try {
|
|
162
|
+
sha = execSync(
|
|
163
|
+
`git rev-parse "${ref}:${repoRelativePath}"`,
|
|
164
|
+
{ cwd: repoRoot, encoding: 'utf-8', stdio: ['pipe', 'pipe', 'pipe'] }
|
|
80
165
|
).trim();
|
|
81
|
-
return sha;
|
|
82
166
|
} catch {
|
|
83
|
-
//
|
|
84
|
-
//
|
|
85
|
-
|
|
86
|
-
|
|
87
|
-
|
|
88
|
-
|
|
89
|
-
|
|
90
|
-
|
|
91
|
-
|
|
92
|
-
|
|
93
|
-
|
|
94
|
-
}
|
|
167
|
+
// ONE PREDICATE ("tracked at ref, in the owning repo"), FOUR DIAGNOSTICS. The decision
|
|
168
|
+
// has already been made above — everything below only explains WHICH WAY the path
|
|
169
|
+
// failed, so DRY holds and each population gets a self-curing instruction.
|
|
170
|
+
return { sha: null, repoRoot, reason: diagnoseUntracked(repoRelativePath, ref, repoRoot, repoLabel) };
|
|
171
|
+
}
|
|
172
|
+
|
|
173
|
+
if (!SHA40.test(sha)) {
|
|
174
|
+
return {
|
|
175
|
+
sha: null, repoRoot,
|
|
176
|
+
reason: `git rev-parse "${ref}:${repoRelativePath}" in ${repoLabel} returned "${sha}", which is not a blob sha`
|
|
177
|
+
};
|
|
95
178
|
}
|
|
179
|
+
return { sha, repoRoot, reason: null };
|
|
96
180
|
}
|
|
97
181
|
|
|
98
|
-
|
|
182
|
+
/**
|
|
183
|
+
* What this gate compares, catches, and does NOT read — printed with the verdict so a
|
|
184
|
+
* reader of a GREEN sees where the green stops, in the gate's own words.
|
|
185
|
+
*/
|
|
186
|
+
export const SOURCE_GATE_BOUNDARY = Object.freeze({
|
|
187
|
+
compares: 'each walked file against `git rev-parse <manifest source ref>:<path relative to THE REPOSITORY THAT OWNS THE FILE>`, where the owner is the nearest enclosing git repository (ManifestLoader::resolveOwningRepo) — the superrepo, a submodule, or an ignored sibling checkout',
|
|
188
|
+
catches: 'any corpus source not tracked at its ref BY ITS OWN REPOSITORY — gitignored scratch, never-committed files, paths absent from an older ref, and paths inside no repository at all',
|
|
189
|
+
does_not_read: 'file CONTENT (see CorpusDenyLint for content/path deny-classes), Pinecone live state, and cross-repo `repo:` sources (refused separately by resolveSourceProvenance). It also does NOT check that a submodule\'s ref matches the commit the superrepo pins at its gitlink: a source is walked at the ref it names in the repository that owns it, so content committed on that branch but not yet pinned by the superrepo is accepted.',
|
|
190
|
+
runs_automatically: 'yes — inside walkCorpus, so every `descix kb corpus sync` passes through it; there is no flag that skips it'
|
|
191
|
+
});
|
|
99
192
|
|
|
100
193
|
/**
|
|
101
194
|
* Resolve ONE source's provenance: which repository, which ref, and the exact commit synced.
|
|
@@ -117,33 +210,62 @@ const SHA40 = /^[0-9a-f]{40}$/i;
|
|
|
117
210
|
export async function resolveSourceProvenance(source, workspaceRoot) {
|
|
118
211
|
const ref = source.ref || DEFAULT_SOURCE_REF;
|
|
119
212
|
const repo = source.repo ?? null;
|
|
120
|
-
//
|
|
121
|
-
//
|
|
122
|
-
//
|
|
213
|
+
// `source_repo` is a LEGACY DISPLAY LABEL that rides into chunk metadata — NOT the
|
|
214
|
+
// ownership decision. Ownership is decided in exactly one place (resolveOwningRepo);
|
|
215
|
+
// this leading-path-segment value is preserved verbatim because rewriting it would
|
|
216
|
+
// re-stamp every existing chunk in every live KB. Do not read it as "which repository
|
|
217
|
+
// owns this file"; for that, ask the owner.
|
|
123
218
|
const source_repo = repo ?? (String(source.path || '').split('/')[0] || 'unknown');
|
|
124
219
|
|
|
125
220
|
if (!repo) {
|
|
221
|
+
// Resolve the commit IN THE REPOSITORY THAT OWNS THE SOURCE — the same one owner the
|
|
222
|
+
// gate consults. Resolving it at the ambient workspace root would record the
|
|
223
|
+
// SUPERREPO's commit for content whose blobs came from a submodule or a sibling repo:
|
|
224
|
+
// two derivations of "where did this content come from" that disagree, which is the
|
|
225
|
+
// mirror-drift this file already refuses elsewhere. A provenance record that names the
|
|
226
|
+
// wrong repository's commit is a FALSE record, not an approximate one.
|
|
227
|
+
const owner = resolveOwningRepo(source.absolutePath);
|
|
228
|
+
if (!owner) {
|
|
229
|
+
throw new Error(
|
|
230
|
+
`Source "${source.path}" resolves to ${source.absolutePath}, which is inside NO git ` +
|
|
231
|
+
`repository. Refusing to record provenance — there is no repository whose ref "${ref}" ` +
|
|
232
|
+
`could describe this content.`
|
|
233
|
+
);
|
|
234
|
+
}
|
|
235
|
+
const provenanceRoot = owner.repoRoot;
|
|
126
236
|
let sha;
|
|
127
237
|
try {
|
|
128
238
|
sha = execSync(`git rev-parse "${ref}^{commit}"`, {
|
|
129
|
-
cwd:
|
|
239
|
+
cwd: provenanceRoot, encoding: 'utf-8', stdio: ['pipe', 'pipe', 'pipe'],
|
|
130
240
|
}).trim();
|
|
131
241
|
} catch {
|
|
132
242
|
throw new Error(
|
|
133
|
-
`Cannot resolve ref "${ref}" to a commit in ${
|
|
134
|
-
`Refusing to record provenance — a sync-state commit of
|
|
135
|
-
`missing one. Check that the ref exists
|
|
243
|
+
`Cannot resolve ref "${ref}" to a commit in ${provenanceRoot} (the repository that owns ` +
|
|
244
|
+
`source "${source.path}") . Refusing to record provenance — a sync-state commit of ` +
|
|
245
|
+
`"unknown" is a FALSE record, not a missing one. Check that the ref exists in THAT ` +
|
|
246
|
+
`repository, which is not necessarily the workspace root ${workspaceRoot}.`
|
|
136
247
|
);
|
|
137
248
|
}
|
|
138
249
|
if (!SHA40.test(sha)) {
|
|
139
250
|
throw new Error(`Ref "${ref}" resolved to "${sha}", which is not a commit sha (source "${source.path}").`);
|
|
140
251
|
}
|
|
141
|
-
|
|
252
|
+
// `repo_root` MAKES THE RECORDED COMMIT SELF-DESCRIBING. Before this, every commit in
|
|
253
|
+
// sync-state was a workspace-root commit, so a reader could resolve it at the workspace
|
|
254
|
+
// root and be right by accident. Now a commit may belong to a submodule or a sibling
|
|
255
|
+
// repo, and the SAME 40 hex characters mean different things depending on which
|
|
256
|
+
// repository you ask. Writing the owning repo beside the commit is what keeps the store
|
|
257
|
+
// homogeneous: a reader resolves the commit WHERE THIS SAYS, and a record written before
|
|
258
|
+
// this field existed is missing it and can be detected rather than silently misread.
|
|
259
|
+
return {
|
|
260
|
+
repo: null, ref, resolved_commit_sha: sha, source_repo,
|
|
261
|
+
repo_root: path.relative(workspaceRoot, provenanceRoot) || '.',
|
|
262
|
+
};
|
|
142
263
|
}
|
|
143
264
|
|
|
144
|
-
// CROSS-REPO: resolve against the remote. Never a local read.
|
|
265
|
+
// CROSS-REPO: resolve against the remote. Never a local read. `repo_root` is null because
|
|
266
|
+
// the content is not in this workspace at all — `repo` already says where it lives.
|
|
145
267
|
if (SHA40.test(ref)) {
|
|
146
|
-
return { repo, ref, resolved_commit_sha: ref.toLowerCase(), source_repo };
|
|
268
|
+
return { repo, ref, resolved_commit_sha: ref.toLowerCase(), source_repo, repo_root: null };
|
|
147
269
|
}
|
|
148
270
|
const remoteUrl = `https://github.com/${repo}.git`;
|
|
149
271
|
let out;
|
|
@@ -167,7 +289,7 @@ export async function resolveSourceProvenance(source, workspaceRoot) {
|
|
|
167
289
|
`Repo "${repo}" has no ref "${ref}" (source "${source.path}"). REFUSING rather than guessing.`
|
|
168
290
|
);
|
|
169
291
|
}
|
|
170
|
-
return { repo, ref, resolved_commit_sha: sha, source_repo };
|
|
292
|
+
return { repo, ref, resolved_commit_sha: sha, source_repo, repo_root: null };
|
|
171
293
|
}
|
|
172
294
|
|
|
173
295
|
/**
|
|
@@ -247,6 +369,9 @@ async function isFile(filePath) {
|
|
|
247
369
|
export async function walkCorpus(manifest, workspaceRoot) {
|
|
248
370
|
const files = [];
|
|
249
371
|
const provenance = [];
|
|
372
|
+
// Every file the source gate refused, collected so ONE error can name them all. A
|
|
373
|
+
// first-throw would hide offenders 2..N and turn one fix into N sync attempts.
|
|
374
|
+
const refused = [];
|
|
250
375
|
let commitSha = 'unknown';
|
|
251
376
|
|
|
252
377
|
// Nested-source dedup (WS-EVP-DESCIX-KB-CLEANUP item 4): a manifest may list both a directory
|
|
@@ -276,6 +401,39 @@ export async function walkCorpus(manifest, workspaceRoot) {
|
|
|
276
401
|
const { absolutePath, ref, tier, doc_type, syncignore } = source;
|
|
277
402
|
const thisSourceRoot = path.resolve(absolutePath);
|
|
278
403
|
|
|
404
|
+
// An IN-REPO source whose path lies in NO repository joins the AGGREGATED refusal rather
|
|
405
|
+
// than throwing. Both outcomes refuse — but throwing would abort on the first offending
|
|
406
|
+
// source and hide every later one, turning one manifest fix into N sync attempts, which
|
|
407
|
+
// is the whole reason this walk aggregates. Measured: `EGPT/` (in no repo) aborted the
|
|
408
|
+
// EVP-EGPT walk before its other sources were ever examined.
|
|
409
|
+
//
|
|
410
|
+
// CROSS-REPO SOURCES ARE DELIBERATELY EXEMPT. Their `absolutePath` is a fiction — the
|
|
411
|
+
// content lives in another repository — so "is it in a repo here" is the wrong question
|
|
412
|
+
// and answering it would MASK the accurate refusal resolveSourceProvenance already owns.
|
|
413
|
+
// Measured: EVP-BEAST's four `repo: eabadir/unk` sources were being reported as "not
|
|
414
|
+
// inside any git repository" instead of "cross-repo content fetch is not implemented".
|
|
415
|
+
if (source.repo === undefined) {
|
|
416
|
+
const exists = await fs.access(absolutePath).then(() => true, () => false);
|
|
417
|
+
if (!exists) {
|
|
418
|
+
refused.push({
|
|
419
|
+
path: source.path, ref, source: source.path,
|
|
420
|
+
reason: `DOES NOT EXIST at ${absolutePath} — the manifest names a path that is not ` +
|
|
421
|
+
`on disk; correct the path or remove this source from the manifest`,
|
|
422
|
+
});
|
|
423
|
+
continue;
|
|
424
|
+
}
|
|
425
|
+
if (!resolveOwningRepo(absolutePath)) {
|
|
426
|
+
refused.push({
|
|
427
|
+
path: source.path, ref, source: source.path,
|
|
428
|
+
reason: `NOT INSIDE ANY GIT REPOSITORY (${absolutePath}) — no repository owns this ` +
|
|
429
|
+
`path, so ref "${ref}" cannot be resolved for it and nothing can vouch for ` +
|
|
430
|
+
`its content; remove this source from the manifest, or put the content in a ` +
|
|
431
|
+
`repository and commit it`,
|
|
432
|
+
});
|
|
433
|
+
continue;
|
|
434
|
+
}
|
|
435
|
+
}
|
|
436
|
+
|
|
279
437
|
// Resolve this source's exact provenance BEFORE reading anything. A cross-repo source that
|
|
280
438
|
// cannot be fetched REFUSES here, naming the repo — it is never silently skipped and never
|
|
281
439
|
// falls back to reading an adjacent working tree.
|
|
@@ -318,13 +476,17 @@ export async function walkCorpus(manifest, workspaceRoot) {
|
|
|
318
476
|
if (ownerSourceRoot(filePath) !== thisSourceRoot) {
|
|
319
477
|
continue;
|
|
320
478
|
}
|
|
321
|
-
//
|
|
479
|
+
// Workspace-relative path is the DISPLAY/metadata spelling only. Git resolution uses
|
|
480
|
+
// the owning repository — see resolveTrackedBlobSha.
|
|
322
481
|
const workspaceRelativePath = path.relative(workspaceRoot, filePath);
|
|
323
482
|
|
|
324
|
-
//
|
|
325
|
-
|
|
483
|
+
// THE SOURCE GATE. A file the OWNING repository does not track at this source's ref
|
|
484
|
+
// is REFUSED by name — never silently chunked from the working tree, and never
|
|
485
|
+
// silently skipped either (a skip on a clean worktree would PURGE its live chunks).
|
|
486
|
+
const { sha: blobSha, reason } = resolveTrackedBlobSha(filePath, ref, workspaceRoot);
|
|
326
487
|
if (!blobSha) {
|
|
327
|
-
|
|
488
|
+
refused.push({ path: workspaceRelativePath, ref, reason, source: source.path });
|
|
489
|
+
continue;
|
|
328
490
|
}
|
|
329
491
|
|
|
330
492
|
// "Which repo does this content belong to" has ONE owner: resolveSourceProvenance.
|
|
@@ -351,7 +513,36 @@ export async function walkCorpus(manifest, workspaceRoot) {
|
|
|
351
513
|
}
|
|
352
514
|
}
|
|
353
515
|
|
|
354
|
-
|
|
516
|
+
// ONE aggregated refusal naming every offender, its ref and its reason. Raised AFTER
|
|
517
|
+
// the full walk so the operator fixes the manifest once rather than N times, and BEFORE
|
|
518
|
+
// any caller can chunk, upsert or purge — the refused files are neither served nor
|
|
519
|
+
// treated as deletions.
|
|
520
|
+
if (refused.length > 0) {
|
|
521
|
+
// The error must be fixable IN ONE ACT from its text alone: it names the manifest
|
|
522
|
+
// FILE to edit and the offending ENTRY within it, not merely the resolved path.
|
|
523
|
+
// `_manifestPath` is owned by ManifestLoader — consume it, never re-derive it.
|
|
524
|
+
const manifestFile = manifest._manifestPath || `(manifest for kb "${manifest.kb_name}")`;
|
|
525
|
+
const lines = refused.map(
|
|
526
|
+
r => ` - ${r.path}\n ref: "${r.ref}" manifest entry: sources[] path "${r.source}"\n ${r.reason}`
|
|
527
|
+
);
|
|
528
|
+
const err = new Error(
|
|
529
|
+
`Corpus source gate FAILED for KB "${manifest.kb_name}": ${refused.length} file(s) are ` +
|
|
530
|
+
`not tracked by git at their manifest ref — refusing to sync.\n` +
|
|
531
|
+
`Manifest to edit: ${manifestFile}\n${lines.join('\n')}\n` +
|
|
532
|
+
`Every corpus source must be committed at its ref. There is no working-tree fallback: ` +
|
|
533
|
+
`content git excludes must never be served as a citable KB source.`
|
|
534
|
+
);
|
|
535
|
+
// The offenders as DATA, so a reporting tool consumes the gate's own verdict instead of
|
|
536
|
+
// re-implementing the predicate or scraping this prose. The message stays the human
|
|
537
|
+
// surface; `refused` is the machine one, and they cannot disagree because there is one
|
|
538
|
+
// decision behind both.
|
|
539
|
+
err.refused = refused;
|
|
540
|
+
err.kb_name = manifest.kb_name;
|
|
541
|
+
err.manifestPath = manifestFile;
|
|
542
|
+
throw err;
|
|
543
|
+
}
|
|
544
|
+
|
|
545
|
+
return { files, commitSha, provenance, gateBoundary: SOURCE_GATE_BOUNDARY };
|
|
355
546
|
}
|
|
356
547
|
|
|
357
|
-
export default { walkCorpus, resolveSourceProvenance };
|
|
548
|
+
export default { walkCorpus, resolveSourceProvenance, resolveTrackedBlobSha, SOURCE_GATE_BOUNDARY };
|