@descix/cli 1.0.1 → 1.0.4
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE.md +671 -0
- package/README.md +43 -4
- package/agent-assets/guides/README.md +2 -2
- package/agent-assets/guides/dynamic-site-pattern.md +29 -14
- package/agent-assets/guides/knowledge-only-pattern.md +5 -5
- package/agent-assets/guides/microservice-pattern.md +48 -4
- package/agent-assets/guides/static-site-pattern.md +8 -0
- package/agent-assets/instructions/app-asset-validation.md +5 -5
- package/agent-assets/instructions/onboarding-flow.md +4 -4
- package/agent-assets/instructions/standard-folder-structure.md +20 -11
- package/agent-assets/reference/app-creation.md +42 -20
- package/agent-assets/reference/architecture.md +24 -22
- package/agent-assets/reference/local-dev.md +230 -67
- package/agent-assets/reference/mcp-integration.md +21 -15
- package/agent-assets/reference/platform-api-consumer.md +244 -0
- package/agent-assets/reference/workspace-config.md +7 -7
- package/agent-assets/rules/descix_mcp.template.mdc +8 -8
- package/bin/descix.js +2559 -1435
- package/bin/kb-deny-lint.mjs +91 -0
- package/bin/mcp-server.js +175 -91
- package/lib/agent-files.js +37 -25
- package/lib/api-client.js +223 -56
- package/lib/auth-guard.js +13 -3
- package/lib/chat-session.js +187 -0
- package/lib/cli-version.js +59 -0
- package/lib/commands/airdrop.js +251 -0
- package/lib/commands/auth.js +254 -72
- package/lib/commands/briefer/errors.js +88 -0
- package/lib/commands/briefer/index.js +319 -0
- package/lib/commands/briefer/sources/canonical-sources.js +129 -0
- package/lib/commands/briefer/sources/entitlements.js +220 -0
- package/lib/commands/briefer/sources/environments.js +205 -0
- package/lib/commands/briefer/sources/identifiers.js +214 -0
- package/lib/commands/briefer/sources/microservice-deploy.js +267 -0
- package/lib/commands/briefer/sources/routing.js +275 -0
- package/lib/commands/briefer/sources/what-is-not.js +223 -0
- package/lib/commands/briefer/stitcher.js +134 -0
- package/lib/commands/briefer/util/source-reader.js +502 -0
- package/lib/commands/config.js +210 -90
- package/lib/commands/corpus.js +978 -0
- package/lib/commands/credits.js +122 -0
- package/lib/commands/health.js +627 -0
- package/lib/commands/kb-list-render.js +32 -0
- package/lib/commands/kb.js +190 -413
- package/lib/commands/mcp.js +5 -5
- package/lib/commands/microservice-restart.js +359 -0
- package/lib/commands/model-config.js +313 -0
- package/lib/commands/retired-kb-sync.js +134 -0
- package/lib/commands/serve.js +26 -1
- package/lib/commands/status.js +23 -51
- package/lib/commands/update.js +36 -107
- package/lib/core/Chunker.js +491 -17
- package/lib/core/CorpusDenyLint.js +326 -0
- package/lib/core/CorpusWalker.js +357 -0
- package/lib/core/Hydrator.js +75 -29
- package/lib/core/ManifestLoader.js +533 -0
- package/lib/core/SiteWalker.js +218 -0
- package/lib/core/Syncer.js +163 -11
- package/lib/diagnostic-log.js +41 -0
- package/lib/environment-report.js +100 -0
- package/lib/global-config.js +4 -1
- package/lib/oauth-client.js +164 -0
- package/lib/origin.js +143 -0
- package/lib/service-api-client.js +95 -0
- package/lib/wallet-environment.js +49 -0
- package/lib/wallet-file.js +16 -2
- package/lib/wizard/setup.js +10 -9
- package/lib/workspace-config.js +380 -235
- package/lib/workspace-identity.js +92 -0
- package/lib/workspace-utils.js +41 -61
- package/package.json +19 -6
- package/templates/agent-claude.md +157 -9
- package/templates/agent-cline.md +68 -8
- package/templates/agent-copilot.md +68 -8
- package/templates/agent-cursor.md +68 -8
- package/templates/alignment-schema.json +20 -0
- package/templates/cloud-brain-hook.sh +20 -0
- package/templates/drive/README.md +1 -1
- package/templates/drive_templates/README.md +2 -2
- package/templates/scaffolds/README.md +6 -10
- package/templates/scaffolds/microservice/SERVICE_README_sdk.md +104 -149
- package/templates/scaffolds/microservice/app.js +51 -10
- package/templates/scaffolds/microservice/app.yaml +2 -2
- package/templates/scaffolds/microservice/defaults-config-dev.json +4 -0
- package/templates/scaffolds/microservice/dev-overrides.example.json +5 -0
- package/templates/scaffolds/microservice/manifest.json +0 -1
- package/templates/scaffolds/microservice/package.json +3 -3
- package/templates/scaffolds/microservice/scripts/register.js +56 -41
- package/templates/scaffolds/microservice/services/mcpClient.js +55 -65
- package/templates/scaffolds/microservice/services/utils.js +20 -238
- package/templates/scaffolds/microservice/templates/SERVICE_README_TEMPLATE.md +24 -0
- package/lib/commands/app-wizard.js +0 -627
- package/lib/commands/folder.js +0 -291
- package/lib/core/PathContext.js +0 -335
- package/templates/scaffolds/microservice/dev-overrides.json.example +0 -5
- package/templates/scaffolds/site/DeSciXAppSDK.js +0 -62
- package/templates/scaffolds/site/README.md +0 -51
- package/templates/scaffolds/site/app.js +0 -22
- package/templates/scaffolds/site/index.html +0 -42
- package/templates/scaffolds/site/styles.css +0 -123
package/lib/commands/kb.js
CHANGED
|
@@ -46,7 +46,7 @@ import * as driveADC from '../google-storage-adc.js';
|
|
|
46
46
|
*/
|
|
47
47
|
export async function runKbPull(apiClient, options) {
|
|
48
48
|
const spinner = ora('Loading workspace configuration...').start();
|
|
49
|
-
|
|
49
|
+
|
|
50
50
|
try {
|
|
51
51
|
// 1. Load WorkspaceConfig (from workspace.json - no searching)
|
|
52
52
|
const workspaceConfig = await WorkspaceConfig.load();
|
|
@@ -56,29 +56,40 @@ export async function runKbPull(apiClient, options) {
|
|
|
56
56
|
|
|
57
57
|
spinner.text = `Pulling KB: ${communityId}/${appId}/${kbId}`;
|
|
58
58
|
|
|
59
|
-
// 3.
|
|
59
|
+
// 3. Handle --folder override: extract folder ID from raw ID or full Drive URL
|
|
60
|
+
let directFolderId = null;
|
|
61
|
+
if (options.folder) {
|
|
62
|
+
const folderInput = options.folder;
|
|
63
|
+
// Handle full Drive URLs: https://drive.google.com/drive/u/0/folders/FOLDER_ID or variants
|
|
64
|
+
const urlMatch = folderInput.match(/\/folders\/([a-zA-Z0-9_-]+)/);
|
|
65
|
+
directFolderId = urlMatch ? urlMatch[1] : folderInput;
|
|
66
|
+
spinner.text = `Pulling KB from override folder: ${directFolderId.substring(0, 12)}...`;
|
|
67
|
+
}
|
|
68
|
+
|
|
69
|
+
// 4. Validate Drive configuration (skip base_folder_id check when --folder is provided)
|
|
60
70
|
const driveConfig = workspaceConfig.driveConfig;
|
|
61
|
-
if (!driveConfig?.base_folder_id) {
|
|
71
|
+
if (!directFolderId && !driveConfig?.base_folder_id) {
|
|
62
72
|
spinner.fail('Drive not configured');
|
|
63
|
-
console.log(chalk.yellow('\n💡
|
|
64
|
-
throw new Error('The base_folder_id is required for KB operations.');
|
|
73
|
+
console.log(chalk.yellow('\n💡 Drive is not linked: set driveConfig.base_folder_id in .descix/workspace.json, or use --folder <id> for a one-time import.\n'));
|
|
74
|
+
throw new Error('The base_folder_id is required for KB operations (or use --folder for one-time import).');
|
|
65
75
|
}
|
|
66
76
|
|
|
67
|
-
//
|
|
77
|
+
// 5. Get paths
|
|
68
78
|
const workspaceRoot = workspaceConfig.getWorkspaceRoot();
|
|
69
79
|
const appPath = workspaceConfig.getAppByAppId(appId)?.absolutePath;
|
|
70
|
-
|
|
71
|
-
//
|
|
80
|
+
|
|
81
|
+
// 6. Delegate to Hydrator with merge mode options
|
|
72
82
|
spinner.text = 'Connecting to Google Drive...';
|
|
73
|
-
|
|
83
|
+
|
|
74
84
|
const result = await hydrateKb({
|
|
75
85
|
workspaceRoot,
|
|
76
86
|
communityId,
|
|
77
87
|
appId,
|
|
78
88
|
kbId,
|
|
79
|
-
baseFolderId: driveConfig.base_folder_id,
|
|
89
|
+
baseFolderId: directFolderId ? null : driveConfig.base_folder_id,
|
|
90
|
+
directFolderId,
|
|
80
91
|
localPath: appPath ? (path.relative(workspaceRoot, appPath) || '.') : `${communityId}/${appId}`
|
|
81
|
-
}, {
|
|
92
|
+
}, {
|
|
82
93
|
verbose: options.verbose,
|
|
83
94
|
mergeMode: options.mergeMode || 'merge',
|
|
84
95
|
dryRun: options.dryRun,
|
|
@@ -140,7 +151,7 @@ export async function runKbPush(apiClient, options) {
|
|
|
140
151
|
const driveConfig = workspaceConfig.driveConfig;
|
|
141
152
|
if (!driveConfig?.base_folder_id) {
|
|
142
153
|
spinner.fail('Drive not configured');
|
|
143
|
-
throw new Error('
|
|
154
|
+
throw new Error('Drive is not linked: set driveConfig.base_folder_id in .descix/workspace.json.');
|
|
144
155
|
}
|
|
145
156
|
|
|
146
157
|
// 4. Get paths
|
|
@@ -221,434 +232,200 @@ export async function runKbPush(apiClient, options) {
|
|
|
221
232
|
}
|
|
222
233
|
}
|
|
223
234
|
|
|
224
|
-
//
|
|
235
|
+
// `kb chunk`, `kb sync`, `kb build`, `kb status` and `kb compare` are REMOVED.
|
|
236
|
+
// runKbChunk / runKbSync / runKbBuild / runKbStatus / runKbCompare are DELETED, not
|
|
237
|
+
// retained-for-later: the one canonical KB sync surface is `descix kb corpus sync`
|
|
238
|
+
// (lib/commands/corpus.js::runCorpusSync). runKbPull/runKbPush above survive because
|
|
239
|
+
// `descix drive pull` / `descix drive push` still call them — they move files to and
|
|
240
|
+
// from Drive and never write Pinecone.
|
|
225
241
|
|
|
242
|
+
// ============ M3: `descix kb doctor` — drift detector (2026-04-20) ============
|
|
226
243
|
/**
|
|
227
|
-
*
|
|
228
|
-
*
|
|
229
|
-
*
|
|
230
|
-
*
|
|
244
|
+
* Compare local sync-state vs live Pinecone vectorCount, and scan the most
|
|
245
|
+
* recent verbose sync log (if any) for per-file 0-chunk warnings. Reports
|
|
246
|
+
* drift direction and exits non-zero when drift exceeds the threshold.
|
|
247
|
+
*
|
|
248
|
+
* WHY THIS EXISTS
|
|
249
|
+
* ---------------
|
|
250
|
+
* `descix kb corpus sync` trusts local sync-state/<KB>.json for delta
|
|
251
|
+
* detection. When the Pinecone index is reset (e.g. dev-env churn), the
|
|
252
|
+
* local state still lists all blob SHAs as "synced", so the next sync
|
|
253
|
+
* skips re-upload and orphan/loss goes undetected until a content
|
|
254
|
+
* retrieval test fails. This command surfaces the drift directly.
|
|
255
|
+
*
|
|
256
|
+
* BEHAVIOUR
|
|
257
|
+
* ---------
|
|
258
|
+
* descix kb doctor -a <app> -k <kb>
|
|
259
|
+
*
|
|
260
|
+
* (a) Queries Pinecone via get_kb_rag_status for vectorCount.
|
|
261
|
+
* (b) Reads {appRoot}/.descix/sync-state/{kb}.json for total_chunks.
|
|
262
|
+
* (c) Reports diff and direction:
|
|
263
|
+
* Pinecone < sync-state → LOST (chunks missing from Pinecone)
|
|
264
|
+
* Pinecone > sync-state → ORPHANS (Pinecone has extra vectors)
|
|
265
|
+
* |drift| / local ≤ threshold → HEALTHY
|
|
266
|
+
* (d) Scans the most recent file matching logs/kb-sync-*.log for
|
|
267
|
+
* '0-chunk' / 'skipped' warnings (these are the signal M1 wired
|
|
268
|
+
* into corpus.js).
|
|
269
|
+
* (e) Exits non-zero on drift above threshold (default: 5%).
|
|
231
270
|
*/
|
|
232
|
-
export async function
|
|
233
|
-
const
|
|
234
|
-
|
|
271
|
+
export async function runKbDoctor(apiClient, options) {
|
|
272
|
+
const { WorkspaceConfig } = await import('../workspace-config.js');
|
|
273
|
+
const workspaceConfig = await WorkspaceConfig.load();
|
|
274
|
+
|
|
275
|
+
const appId = options.app;
|
|
276
|
+
const kbName = options.kb;
|
|
277
|
+
if (!appId) throw new Error('-a, --app <id> is required');
|
|
278
|
+
if (!kbName) throw new Error('-k, --kb <name> is required');
|
|
279
|
+
|
|
280
|
+
const appMeta = workspaceConfig.getAppByAppId(appId);
|
|
281
|
+
if (!appMeta) throw new Error(`App '${appId}' not found in workspace.json`);
|
|
282
|
+
const appRoot = appMeta.absolutePath;
|
|
283
|
+
const communityId = appMeta.communityId || appMeta.community_id || null;
|
|
284
|
+
|
|
285
|
+
// Drift threshold (fraction). CEO guidance 2026-04-20: ~5% is reasonable.
|
|
286
|
+
const DRIFT_THRESHOLD = Number.isFinite(Number(options.threshold))
|
|
287
|
+
? Number(options.threshold) : 0.05;
|
|
288
|
+
|
|
289
|
+
const scope = communityId ? `${communityId}/${appId}/${kbName}` : `${appId}/${kbName}`;
|
|
290
|
+
console.log(chalk.cyan(`\n🩺 kb doctor — ${scope}\n`));
|
|
291
|
+
|
|
292
|
+
// --live : compute vectorCount from the TRUE live Pinecone scope (id-prefix
|
|
293
|
+
// enumeration), bypassing the cached rag_vector_count counter that
|
|
294
|
+
// LIES after an interrupted op.
|
|
295
|
+
// --reconcile: compute the live count AND write it back to rag_vector_count so the
|
|
296
|
+
// fast cached read is truthful again. Implies --live truth.
|
|
297
|
+
const live = !!options.live || !!options.reconcile;
|
|
298
|
+
const reconcile = !!options.reconcile;
|
|
299
|
+
|
|
300
|
+
// (a) Pinecone vector count (cached by default; live/reconciled on request)
|
|
301
|
+
let vectorCount = null;
|
|
302
|
+
let ragStatus = null;
|
|
235
303
|
try {
|
|
236
|
-
|
|
237
|
-
|
|
238
|
-
|
|
239
|
-
|
|
240
|
-
|
|
241
|
-
|
|
242
|
-
|
|
243
|
-
|
|
244
|
-
|
|
245
|
-
|
|
246
|
-
const appPath = workspaceConfig.getAppByAppId(appId)?.absolutePath;
|
|
247
|
-
|
|
248
|
-
// 4. Delegate to Chunker
|
|
249
|
-
const chunkOptions = {
|
|
250
|
-
maxChunkSize: parseInt(options.chunkSize || 2000),
|
|
251
|
-
overlapSize: parseInt(options.overlap || 500),
|
|
252
|
-
verbose: options.verbose
|
|
253
|
-
};
|
|
254
|
-
|
|
255
|
-
if (options.verbose) {
|
|
256
|
-
console.log(chalk.gray(` Chunk size: ${chunkOptions.maxChunkSize}`));
|
|
257
|
-
console.log(chalk.gray(` Overlap: ${chunkOptions.overlapSize}`));
|
|
258
|
-
}
|
|
259
|
-
|
|
260
|
-
const result = await processKb({
|
|
261
|
-
workspaceRoot,
|
|
262
|
-
communityId,
|
|
263
|
-
appId,
|
|
264
|
-
kbId,
|
|
265
|
-
localPath: appPath ? (path.relative(workspaceRoot, appPath) || '.') : `${communityId}/${appId}`
|
|
266
|
-
}, chunkOptions);
|
|
267
|
-
|
|
268
|
-
spinner.succeed(`Generated ${result.totalChunks} chunks from ${result.files} files`);
|
|
269
|
-
|
|
270
|
-
// Show next steps
|
|
271
|
-
if (!options.quiet) {
|
|
272
|
-
console.log(chalk.cyan('\n📋 Next steps:'));
|
|
273
|
-
console.log(chalk.gray(' Run "descix kb sync" to push chunks to Pinecone\n'));
|
|
304
|
+
const res = await apiClient.invoke('get_kb_rag_status', {
|
|
305
|
+
app_id: appId,
|
|
306
|
+
kb_id: kbName,
|
|
307
|
+
live,
|
|
308
|
+
reconcile
|
|
309
|
+
}, { allowGuest: false });
|
|
310
|
+
ragStatus = res.message || res;
|
|
311
|
+
vectorCount = ragStatus?.vectorCount;
|
|
312
|
+
if (typeof vectorCount !== 'number') {
|
|
313
|
+
throw new Error(`get_kb_rag_status returned no vectorCount: ${JSON.stringify(ragStatus)}`);
|
|
274
314
|
}
|
|
275
|
-
|
|
276
|
-
|
|
277
|
-
|
|
278
|
-
} catch (error) {
|
|
279
|
-
spinner.fail('Chunk failed');
|
|
280
|
-
throw error;
|
|
315
|
+
} catch (err) {
|
|
316
|
+
console.log(chalk.red(` ✗ get_kb_rag_status failed: ${err.message}`));
|
|
317
|
+
throw err;
|
|
281
318
|
}
|
|
282
|
-
}
|
|
283
|
-
|
|
284
|
-
// ============ Sync Command ============
|
|
285
319
|
|
|
286
|
-
|
|
287
|
-
|
|
288
|
-
|
|
289
|
-
|
|
290
|
-
|
|
291
|
-
|
|
292
|
-
|
|
293
|
-
|
|
294
|
-
|
|
295
|
-
|
|
296
|
-
|
|
297
|
-
|
|
298
|
-
|
|
299
|
-
|
|
300
|
-
|
|
301
|
-
|
|
302
|
-
// 1. Load WorkspaceConfig
|
|
303
|
-
const workspaceConfig = await WorkspaceConfig.load();
|
|
304
|
-
|
|
305
|
-
// 2. Resolve context
|
|
306
|
-
const { communityId, appId, kbId } = workspaceConfig.requireContext(options);
|
|
307
|
-
|
|
308
|
-
spinner.text = `Syncing KB: ${communityId}/${appId}/${kbId}`;
|
|
309
|
-
|
|
310
|
-
// 3. Get paths
|
|
311
|
-
const workspaceRoot = workspaceConfig.getWorkspaceRoot();
|
|
312
|
-
const appPath = workspaceConfig.getAppByAppId(appId)?.absolutePath;
|
|
313
|
-
|
|
314
|
-
// 4. Delegate to Syncer
|
|
315
|
-
const result = await syncKb(apiClient, {
|
|
316
|
-
workspaceRoot,
|
|
317
|
-
communityId,
|
|
318
|
-
appId,
|
|
319
|
-
kbId,
|
|
320
|
-
localPath: appPath ? (path.relative(workspaceRoot, appPath) || '.') : `${communityId}/${appId}`
|
|
321
|
-
}, {
|
|
322
|
-
verbose: options.verbose,
|
|
323
|
-
onProgress: (msg) => { spinner.text = msg; }
|
|
324
|
-
});
|
|
325
|
-
|
|
326
|
-
if (result.synced > 0 || result.deleted > 0) {
|
|
327
|
-
spinner.succeed(`Synced ${result.synced} chunks (${result.deleted} deleted)`);
|
|
328
|
-
} else {
|
|
329
|
-
spinner.succeed('Already in sync');
|
|
320
|
+
if (reconcile) {
|
|
321
|
+
console.log(chalk.cyan(
|
|
322
|
+
` ⟳ Reconciled cached count: ${ragStatus.reconcileBefore} → ${ragStatus.reconcileAfter} ` +
|
|
323
|
+
`(${ragStatus.reconcileAfter - ragStatus.reconcileBefore >= 0 ? '+' : ''}${ragStatus.reconcileAfter - ragStatus.reconcileBefore})\n`
|
|
324
|
+
));
|
|
325
|
+
}
|
|
326
|
+
if (live) {
|
|
327
|
+
const src = ragStatus.source === 'live' ? chalk.green('LIVE (Pinecone)') : chalk.yellow(ragStatus.source);
|
|
328
|
+
console.log(` Count source : ${src}`);
|
|
329
|
+
if (typeof ragStatus.cachedVectorCount === 'number') {
|
|
330
|
+
const cacheDrift = ragStatus.drift;
|
|
331
|
+
console.log(
|
|
332
|
+
` Cached counter : ${chalk.white(ragStatus.cachedVectorCount)} ` +
|
|
333
|
+
`(live − cache = ${cacheDrift >= 0 ? '+' : ''}${cacheDrift})` +
|
|
334
|
+
(cacheDrift === 0 ? chalk.green(' ✓ truthful') : chalk.yellow(' ⚠ cache drifted; run --reconcile'))
|
|
335
|
+
);
|
|
330
336
|
}
|
|
331
|
-
|
|
332
|
-
return result;
|
|
333
|
-
|
|
334
|
-
} catch (error) {
|
|
335
|
-
spinner.fail('Sync failed');
|
|
336
|
-
throw error;
|
|
337
337
|
}
|
|
338
|
-
}
|
|
339
|
-
|
|
340
|
-
// ============ Build Command (Combined) ============
|
|
341
338
|
|
|
342
|
-
|
|
343
|
-
|
|
344
|
-
|
|
345
|
-
|
|
346
|
-
* 1. Check staging - if files exist, push to Drive first
|
|
347
|
-
* 2. Pull from Drive with merge mode
|
|
348
|
-
* 3. Chunk local files
|
|
349
|
-
* 4. Sync to Pinecone
|
|
350
|
-
*
|
|
351
|
-
* @param {DeSciXApiClient} apiClient - API client
|
|
352
|
-
* @param {Object} options - Combined options for all steps
|
|
353
|
-
* @param {boolean} options.interactive - Enable interactive prompts
|
|
354
|
-
* @param {boolean} options.skipStaging - Skip staging push step
|
|
355
|
-
* @param {string} options.mergeMode - 'merge' | 'overwrite' | 'force-overwrite'
|
|
356
|
-
* @param {boolean} options.dryRun - Show what would happen
|
|
357
|
-
*/
|
|
358
|
-
export async function runKbBuild(apiClient, options) {
|
|
359
|
-
const { interactive = false, skipStaging = false, dryRun = false } = options;
|
|
360
|
-
|
|
361
|
-
console.log(chalk.cyan('\n=== KB Build Pipeline ===\n'));
|
|
362
|
-
|
|
363
|
-
if (dryRun) {
|
|
364
|
-
console.log(chalk.yellow('DRY RUN MODE - No changes will be made\n'));
|
|
365
|
-
}
|
|
366
|
-
|
|
339
|
+
// (b) Local sync-state total_chunks
|
|
340
|
+
const syncStatePath = path.join(appRoot, '.descix', 'sync-state', `${kbName}.json`);
|
|
341
|
+
let localChunks = null;
|
|
342
|
+
let syncStateRaw = null;
|
|
367
343
|
try {
|
|
368
|
-
|
|
369
|
-
|
|
370
|
-
|
|
371
|
-
|
|
372
|
-
|
|
373
|
-
const localPath = appPath ? (path.relative(workspaceRoot, appPath) || '.') : `${communityId}/${appId}`;
|
|
374
|
-
const stagingDir = path.join(workspaceRoot, localPath, 'kb', 'staging');
|
|
375
|
-
|
|
376
|
-
// Show which app we're building
|
|
377
|
-
console.log(chalk.white(`Target: ${chalk.cyan(communityId)}/${chalk.cyan(appId)}/${chalk.gray(kbId)}\n`));
|
|
378
|
-
|
|
379
|
-
let stepNum = 1;
|
|
380
|
-
const totalSteps = skipStaging ? 3 : 4;
|
|
381
|
-
|
|
382
|
-
// Step 1: Check and push staging (unless skipped)
|
|
383
|
-
if (!skipStaging) {
|
|
384
|
-
const stagingCheck = await checkStagingFiles(stagingDir);
|
|
385
|
-
|
|
386
|
-
if (stagingCheck.hasFiles) {
|
|
387
|
-
console.log(chalk.white(`[${stepNum}/${totalSteps}] Push staging files to Drive`));
|
|
388
|
-
console.log(chalk.gray(` Found ${stagingCheck.count} file(s) in staging:`));
|
|
389
|
-
stagingCheck.files.forEach(f => console.log(chalk.gray(` - ${f}`)));
|
|
390
|
-
console.log('');
|
|
391
|
-
|
|
392
|
-
await runKbPush(apiClient, {
|
|
393
|
-
...options,
|
|
394
|
-
quiet: true,
|
|
395
|
-
onConflict: interactive ? undefined : 'overwrite'
|
|
396
|
-
});
|
|
397
|
-
console.log('');
|
|
398
|
-
} else {
|
|
399
|
-
console.log(chalk.gray(`[${stepNum}/${totalSteps}] No staging files to push\n`));
|
|
400
|
-
}
|
|
401
|
-
stepNum++;
|
|
344
|
+
const raw = await fs.readFile(syncStatePath, 'utf-8');
|
|
345
|
+
syncStateRaw = JSON.parse(raw);
|
|
346
|
+
localChunks = syncStateRaw?.total_chunks;
|
|
347
|
+
if (typeof localChunks !== 'number') {
|
|
348
|
+
throw new Error(`sync-state has no total_chunks: ${syncStatePath}`);
|
|
402
349
|
}
|
|
403
|
-
|
|
404
|
-
|
|
405
|
-
|
|
406
|
-
|
|
407
|
-
|
|
408
|
-
const mergeMode = options.mergeMode || 'merge';
|
|
409
|
-
if (mergeMode !== 'force-overwrite') {
|
|
410
|
-
console.log(chalk.gray(` Mode: ${mergeMode} (use --force-overwrite to replace all)\n`));
|
|
350
|
+
} catch (err) {
|
|
351
|
+
if (err.code === 'ENOENT') {
|
|
352
|
+
console.log(chalk.yellow(` ⚠ No local sync-state at ${syncStatePath}`));
|
|
353
|
+
console.log(chalk.gray(` Run 'descix kb corpus sync -a ${appId} -k ${kbName}' first.`));
|
|
354
|
+
process.exit(2);
|
|
411
355
|
}
|
|
412
|
-
|
|
413
|
-
await runKbPull(apiClient, {
|
|
414
|
-
...options,
|
|
415
|
-
quiet: true,
|
|
416
|
-
mergeMode
|
|
417
|
-
});
|
|
418
|
-
console.log('');
|
|
419
|
-
stepNum++;
|
|
420
|
-
|
|
421
|
-
// Step 3: Chunk
|
|
422
|
-
console.log(chalk.white(`[${stepNum}/${totalSteps}] Generate chunks`));
|
|
423
|
-
await runKbChunk({ ...options, quiet: true });
|
|
424
|
-
console.log('');
|
|
425
|
-
stepNum++;
|
|
426
|
-
|
|
427
|
-
// Step 4: Sync
|
|
428
|
-
console.log(chalk.white(`[${stepNum}/${totalSteps}] Sync to Pinecone`));
|
|
429
|
-
await runKbSync(apiClient, { ...options, quiet: true });
|
|
430
|
-
|
|
431
|
-
console.log(chalk.green('\n✅ KB build complete!\n'));
|
|
432
|
-
|
|
433
|
-
} catch (error) {
|
|
434
|
-
console.error(chalk.red(`\n❌ ${error.message}\n`));
|
|
435
|
-
throw error;
|
|
356
|
+
throw err;
|
|
436
357
|
}
|
|
437
|
-
}
|
|
438
358
|
|
|
439
|
-
//
|
|
440
|
-
|
|
441
|
-
|
|
442
|
-
|
|
443
|
-
|
|
444
|
-
|
|
445
|
-
|
|
446
|
-
|
|
447
|
-
|
|
448
|
-
|
|
449
|
-
|
|
359
|
+
// (c) Report
|
|
360
|
+
const delta = vectorCount - localChunks;
|
|
361
|
+
const ratio = localChunks === 0 ? Infinity : Math.abs(delta) / localChunks;
|
|
362
|
+
const direction = delta === 0 ? 'EXACT'
|
|
363
|
+
: delta < 0 ? 'LOST (Pinecone missing chunks)'
|
|
364
|
+
: 'ORPHANS (Pinecone has extra vectors)';
|
|
365
|
+
const healthy = Math.abs(delta) / Math.max(localChunks, 1) <= DRIFT_THRESHOLD;
|
|
366
|
+
|
|
367
|
+
console.log(` Pinecone vectorCount : ${chalk.white(vectorCount)}`);
|
|
368
|
+
console.log(` Local total_chunks : ${chalk.white(localChunks)}`);
|
|
369
|
+
console.log(` Drift : ${chalk.white((delta >= 0 ? '+' : '') + delta)} (${(ratio * 100).toFixed(1)}%)`);
|
|
370
|
+
console.log(` Direction : ${delta === 0 ? chalk.green(direction) : (healthy ? chalk.yellow(direction) : chalk.red(direction))}`);
|
|
371
|
+
console.log(` Index : ${ragStatus.indexName || '(unknown)'}`);
|
|
372
|
+
console.log(` Last sync (server) : ${ragStatus.lastSync || '(unknown)'}`);
|
|
373
|
+
console.log(` Last sync (local) : ${syncStateRaw.timestamp || '(unknown)'}`);
|
|
374
|
+
|
|
375
|
+
// (d) Scan most recent sync log for 0-chunk / skipped warnings
|
|
376
|
+
const logsDir = path.join(appRoot, '.descix', 'logs');
|
|
377
|
+
let recentLog = null;
|
|
378
|
+
let warnings = [];
|
|
450
379
|
try {
|
|
451
|
-
|
|
452
|
-
const
|
|
453
|
-
|
|
454
|
-
|
|
455
|
-
|
|
456
|
-
|
|
457
|
-
|
|
458
|
-
|
|
459
|
-
|
|
460
|
-
communityId,
|
|
461
|
-
appId,
|
|
462
|
-
kbId,
|
|
463
|
-
localPath: appPath ? (path.relative(workspaceRoot, appPath) || '.') : `${communityId}/${appId}`
|
|
464
|
-
});
|
|
465
|
-
|
|
466
|
-
spinner.stop();
|
|
467
|
-
|
|
468
|
-
console.log(chalk.cyan(`\n📊 KB Status: ${communityId}/${appId}/${kbId}\n`));
|
|
469
|
-
console.log(chalk.white(` Local chunks: ${status.local}`));
|
|
470
|
-
console.log(chalk.white(` Remote chunks: ${status.remote}`));
|
|
471
|
-
|
|
472
|
-
if (status.inSync) {
|
|
473
|
-
console.log(chalk.green('\n ✓ In sync\n'));
|
|
474
|
-
} else {
|
|
475
|
-
console.log(chalk.yellow('\n ⚠ Out of sync:'));
|
|
476
|
-
if (status.toUpload > 0) {
|
|
477
|
-
console.log(chalk.gray(` - ${status.toUpload} chunks to upload`));
|
|
478
|
-
}
|
|
479
|
-
if (status.toDelete > 0) {
|
|
480
|
-
console.log(chalk.gray(` - ${status.toDelete} chunks to delete`));
|
|
380
|
+
const entries = await fs.readdir(logsDir);
|
|
381
|
+
const syncLogs = entries.filter(f => /^kb-sync-.*\.log$/.test(f)).sort();
|
|
382
|
+
if (syncLogs.length > 0) {
|
|
383
|
+
recentLog = path.join(logsDir, syncLogs[syncLogs.length - 1]);
|
|
384
|
+
const content = await fs.readFile(recentLog, 'utf-8');
|
|
385
|
+
for (const line of content.split('\n')) {
|
|
386
|
+
if (/0-chunk|⚠ skipped/.test(line)) {
|
|
387
|
+
warnings.push(line.trim());
|
|
388
|
+
}
|
|
481
389
|
}
|
|
482
|
-
console.log(chalk.gray('\n Run "descix kb sync" to synchronize\n'));
|
|
483
390
|
}
|
|
484
|
-
|
|
485
|
-
|
|
486
|
-
|
|
487
|
-
} catch (error) {
|
|
488
|
-
spinner.fail('Status check failed');
|
|
489
|
-
throw error;
|
|
391
|
+
} catch {
|
|
392
|
+
// logs dir may not exist — not fatal
|
|
490
393
|
}
|
|
491
|
-
}
|
|
492
|
-
|
|
493
|
-
// ============ Compare Command ============
|
|
494
|
-
|
|
495
|
-
/**
|
|
496
|
-
* Show file-level deltas between local and Drive
|
|
497
|
-
*
|
|
498
|
-
* @param {DeSciXApiClient|null} apiClient - API client (unused)
|
|
499
|
-
* @param {Object} options - { community, app, kb, staging, local }
|
|
500
|
-
*/
|
|
501
|
-
export async function runKbCompare(apiClient, options) {
|
|
502
|
-
const spinner = ora('Loading workspace configuration...').start();
|
|
503
|
-
|
|
504
|
-
try {
|
|
505
|
-
// Load WorkspaceConfig
|
|
506
|
-
const workspaceConfig = await WorkspaceConfig.load();
|
|
507
|
-
const { communityId, appId, kbId } = workspaceConfig.requireContext(options);
|
|
508
|
-
|
|
509
|
-
const driveConfig = workspaceConfig.driveConfig;
|
|
510
|
-
if (!driveConfig?.base_folder_id) {
|
|
511
|
-
spinner.fail('Drive not configured');
|
|
512
|
-
throw new Error('Run "descix setup --dev" first to link Drive.');
|
|
513
|
-
}
|
|
514
394
|
|
|
515
|
-
|
|
516
|
-
|
|
517
|
-
|
|
518
|
-
|
|
519
|
-
|
|
520
|
-
|
|
521
|
-
|
|
522
|
-
|
|
523
|
-
// Get Drive KB folder
|
|
524
|
-
const kbPath = `${communityId}/${appId}/kb/${kbId}`;
|
|
525
|
-
const kbDriveFolderId = await driveADC.findFolderByPath(driveConfig.base_folder_id, kbPath);
|
|
526
|
-
|
|
527
|
-
spinner.stop();
|
|
528
|
-
|
|
529
|
-
console.log(chalk.cyan(`\n📊 KB Compare: ${communityId}/${appId}/${kbId}\n`));
|
|
530
|
-
|
|
531
|
-
// Compare staging → Drive
|
|
532
|
-
if (options.staging !== false) {
|
|
533
|
-
const stagingDir = path.join(workspaceRoot, localPath, 'kb', 'staging');
|
|
534
|
-
const stagingCheck = await checkStagingFiles(stagingDir);
|
|
535
|
-
|
|
536
|
-
console.log(chalk.white('Staging → Drive:'));
|
|
537
|
-
|
|
538
|
-
if (!stagingCheck.hasFiles) {
|
|
539
|
-
console.log(chalk.gray(' (no files in staging)\n'));
|
|
540
|
-
} else {
|
|
541
|
-
// Get Drive files for comparison
|
|
542
|
-
const driveFiles = kbDriveFolderId
|
|
543
|
-
? await driveADC.listDriveFiles(kbDriveFolderId)
|
|
544
|
-
: [];
|
|
545
|
-
const driveFileNames = new Set(driveFiles.map(f => f.name.toLowerCase()));
|
|
546
|
-
|
|
547
|
-
for (const fileName of stagingCheck.files) {
|
|
548
|
-
const existsInDrive = driveFileNames.has(fileName.toLowerCase());
|
|
549
|
-
if (existsInDrive) {
|
|
550
|
-
console.log(chalk.yellow(` ~ ${fileName}`) + chalk.gray(' (would overwrite)'));
|
|
551
|
-
} else {
|
|
552
|
-
console.log(chalk.green(` + ${fileName}`) + chalk.gray(' (to upload)'));
|
|
553
|
-
}
|
|
554
|
-
}
|
|
555
|
-
console.log('');
|
|
395
|
+
if (recentLog) {
|
|
396
|
+
console.log(`\n Most recent sync log : ${path.relative(process.cwd(), recentLog)}`);
|
|
397
|
+
if (warnings.length > 0) {
|
|
398
|
+
console.log(chalk.yellow(` Per-file warnings (${warnings.length}):`));
|
|
399
|
+
warnings.slice(0, 10).forEach(w => console.log(chalk.yellow(` ${w}`)));
|
|
400
|
+
if (warnings.length > 10) {
|
|
401
|
+
console.log(chalk.gray(` ...${warnings.length - 10} more`));
|
|
556
402
|
}
|
|
403
|
+
} else {
|
|
404
|
+
console.log(chalk.gray(` No 0-chunk / skipped warnings in most recent log.`));
|
|
557
405
|
}
|
|
558
|
-
|
|
559
|
-
|
|
560
|
-
|
|
561
|
-
|
|
562
|
-
|
|
563
|
-
|
|
564
|
-
|
|
565
|
-
|
|
566
|
-
|
|
567
|
-
|
|
568
|
-
|
|
569
|
-
|
|
570
|
-
|
|
571
|
-
|
|
572
|
-
} catch {
|
|
573
|
-
// KB directory doesn't exist
|
|
574
|
-
}
|
|
575
|
-
|
|
576
|
-
// Get Drive files
|
|
577
|
-
const driveFiles = kbDriveFolderId
|
|
578
|
-
? await driveADC.listDriveFiles(kbDriveFolderId)
|
|
579
|
-
: [];
|
|
580
|
-
|
|
581
|
-
// Load metadata for hash comparison
|
|
582
|
-
let metadata = { files: {} };
|
|
583
|
-
try {
|
|
584
|
-
const metaPath = path.join(kbDir, '.descix_metadata.json');
|
|
585
|
-
const metaContent = await fs.readFile(metaPath, 'utf-8');
|
|
586
|
-
metadata = JSON.parse(metaContent);
|
|
587
|
-
} catch {
|
|
588
|
-
// No metadata
|
|
589
|
-
}
|
|
590
|
-
|
|
591
|
-
const localSet = new Set(localFiles.map(f => f.toLowerCase()));
|
|
592
|
-
const driveSet = new Set(driveFiles.map(f => f.name.toLowerCase()));
|
|
593
|
-
|
|
594
|
-
let hasDeltas = false;
|
|
595
|
-
|
|
596
|
-
// Files in Drive but not local
|
|
597
|
-
for (const driveFile of driveFiles) {
|
|
598
|
-
const localName = driveFile.name.toLowerCase();
|
|
599
|
-
if (!localSet.has(localName)) {
|
|
600
|
-
console.log(chalk.cyan(` ↓ ${driveFile.name}`) + chalk.gray(' (Drive only)'));
|
|
601
|
-
hasDeltas = true;
|
|
602
|
-
}
|
|
603
|
-
}
|
|
604
|
-
|
|
605
|
-
// Files in local
|
|
606
|
-
for (const localFile of localFiles) {
|
|
607
|
-
const meta = metadata.files?.[localFile];
|
|
608
|
-
|
|
609
|
-
if (!driveSet.has(localFile.toLowerCase())) {
|
|
610
|
-
// Local only
|
|
611
|
-
console.log(chalk.magenta(` * ${localFile}`) + chalk.gray(' (local only)'));
|
|
612
|
-
hasDeltas = true;
|
|
613
|
-
} else if (meta?.drive_modified) {
|
|
614
|
-
// In both - check if in sync
|
|
615
|
-
const driveFile = driveFiles.find(f => f.name.toLowerCase() === localFile.toLowerCase());
|
|
616
|
-
if (driveFile && driveFile.modifiedTime !== meta.drive_modified) {
|
|
617
|
-
console.log(chalk.yellow(` ↻ ${localFile}`) + chalk.gray(' (Drive updated)'));
|
|
618
|
-
hasDeltas = true;
|
|
619
|
-
} else {
|
|
620
|
-
console.log(chalk.green(` = ${localFile}`) + chalk.gray(' (in sync)'));
|
|
621
|
-
}
|
|
622
|
-
} else {
|
|
623
|
-
console.log(chalk.gray(` ? ${localFile}`) + chalk.gray(' (unknown sync state)'));
|
|
624
|
-
hasDeltas = true;
|
|
625
|
-
}
|
|
626
|
-
}
|
|
627
|
-
|
|
628
|
-
if (!hasDeltas && localFiles.length === 0 && driveFiles.length === 0) {
|
|
629
|
-
console.log(chalk.gray(' (no files)\n'));
|
|
630
|
-
} else {
|
|
631
|
-
console.log('');
|
|
632
|
-
}
|
|
406
|
+
} else {
|
|
407
|
+
console.log(chalk.gray(`\n No sync log found at ${logsDir} (logs are optional).`));
|
|
408
|
+
}
|
|
409
|
+
|
|
410
|
+
// (e) Exit
|
|
411
|
+
console.log();
|
|
412
|
+
if (!healthy) {
|
|
413
|
+
console.log(chalk.red(` ✗ DRIFT detected: ${(ratio * 100).toFixed(1)}% exceeds ${(DRIFT_THRESHOLD * 100).toFixed(1)}% threshold.\n`));
|
|
414
|
+
if (delta < 0) {
|
|
415
|
+
console.log(chalk.gray(` Recommended: invalidate sync-state and re-sync:`));
|
|
416
|
+
console.log(chalk.gray(` rm "${syncStatePath}"`));
|
|
417
|
+
console.log(chalk.gray(` descix kb corpus sync -a ${appId} -k ${kbName}\n`));
|
|
418
|
+
} else {
|
|
419
|
+
console.log(chalk.gray(` Recommended: clear stale vectors and re-sync to rebuild Pinecone from local state.\n`));
|
|
633
420
|
}
|
|
634
|
-
|
|
635
|
-
// Legend
|
|
636
|
-
console.log(chalk.gray('Legend: + new ~ overwrite ↓ Drive only * local only = in sync ↻ updated\n'));
|
|
637
|
-
|
|
638
|
-
return { communityId, appId, kbId };
|
|
639
|
-
|
|
640
|
-
} catch (error) {
|
|
641
|
-
spinner.fail('Compare failed');
|
|
642
|
-
throw error;
|
|
421
|
+
process.exit(1);
|
|
643
422
|
}
|
|
423
|
+
|
|
424
|
+
console.log(chalk.green(` ✓ HEALTHY (drift within ${(DRIFT_THRESHOLD * 100).toFixed(1)}% threshold)\n`));
|
|
644
425
|
}
|
|
645
426
|
|
|
646
427
|
export default {
|
|
647
428
|
runKbPull,
|
|
648
429
|
runKbPush,
|
|
649
|
-
|
|
650
|
-
runKbSync,
|
|
651
|
-
runKbBuild,
|
|
652
|
-
runKbStatus,
|
|
653
|
-
runKbCompare
|
|
430
|
+
runKbDoctor
|
|
654
431
|
};
|