@karmaniverous/jeeves-watcher 0.15.2 → 0.16.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/cli/jeeves-watcher/index.js +765 -8314
- package/dist/index.d.ts +30 -2
- package/dist/index.js +446 -52
- package/package.json +15 -15
package/dist/index.js
CHANGED
|
@@ -1,4 +1,5 @@
|
|
|
1
1
|
import { join, dirname, resolve, relative, extname, basename, isAbsolute } from 'node:path';
|
|
2
|
+
import { createConfigQueryHandler as createConfigQueryHandler$1, createStatusHandler, getBindAddress, postJson, fetchJson } from '@karmaniverous/jeeves';
|
|
2
3
|
import Fastify from 'fastify';
|
|
3
4
|
import { readdir, stat, writeFile, readFile } from 'node:fs/promises';
|
|
4
5
|
import { parallel, capitalize, title, camel, snake, dash, isEqual, get, omit } from 'radash';
|
|
@@ -9,7 +10,6 @@ import { z, ZodError } from 'zod';
|
|
|
9
10
|
import { jsonMapMapSchema, JsonMap } from '@karmaniverous/jsonmap';
|
|
10
11
|
import Ajv from 'ajv';
|
|
11
12
|
import addFormats from 'ajv-formats';
|
|
12
|
-
import { createConfigQueryHandler as createConfigQueryHandler$1 } from '@karmaniverous/jeeves';
|
|
13
13
|
import { pathToFileURL, fileURLToPath } from 'node:url';
|
|
14
14
|
import Handlebars from 'handlebars';
|
|
15
15
|
import dayjs from 'dayjs';
|
|
@@ -33,6 +33,7 @@ import * as cheerio from 'cheerio';
|
|
|
33
33
|
import mammoth from 'mammoth';
|
|
34
34
|
import { MarkdownTextSplitter, RecursiveCharacterTextSplitter } from '@langchain/textsplitters';
|
|
35
35
|
import { QdrantClient } from '@qdrant/js-client-rest';
|
|
36
|
+
import { createRequire } from 'node:module';
|
|
36
37
|
|
|
37
38
|
/**
|
|
38
39
|
* @module gitignore
|
|
@@ -755,7 +756,7 @@ async function executeReindex(deps, scope, path, dryRun = false) {
|
|
|
755
756
|
let plan;
|
|
756
757
|
if (scope === 'prune') {
|
|
757
758
|
deps.queue?.pause();
|
|
758
|
-
await deps.queue?.
|
|
759
|
+
await deps.queue?.waitForIdleWorkers();
|
|
759
760
|
let pruneResult;
|
|
760
761
|
try {
|
|
761
762
|
pruneResult = await computePrunePlan(deps);
|
|
@@ -3331,7 +3332,7 @@ function buildFacetSchema(rules, mergeOptions) {
|
|
|
3331
3332
|
* @returns Fastify route handler (plain return, compatible with `withCache`).
|
|
3332
3333
|
*/
|
|
3333
3334
|
function createFacetsHandler(deps) {
|
|
3334
|
-
const { getConfig, valuesManager, configDir } = deps;
|
|
3335
|
+
const { getConfig, valuesManager, configDir, logger } = deps;
|
|
3335
3336
|
let cached;
|
|
3336
3337
|
return () => {
|
|
3337
3338
|
const config = getConfig();
|
|
@@ -3339,10 +3340,16 @@ function createFacetsHandler(deps) {
|
|
|
3339
3340
|
globalSchemas: config.schemas,
|
|
3340
3341
|
configDir,
|
|
3341
3342
|
};
|
|
3342
|
-
// Rebuild schema cache if rules changed
|
|
3343
|
+
// Rebuild schema cache if rules changed (#159: catch schema errors gracefully)
|
|
3343
3344
|
const currentHash = computeRulesHash(config.inferenceRules);
|
|
3344
3345
|
if (!cached || cached.rulesHash !== currentHash) {
|
|
3345
|
-
|
|
3346
|
+
try {
|
|
3347
|
+
cached = buildFacetSchema(config.inferenceRules, mergeOptions);
|
|
3348
|
+
}
|
|
3349
|
+
catch (err) {
|
|
3350
|
+
logger?.error({ err }, 'facets: failed to build facet schema from inference rules; returning empty facets');
|
|
3351
|
+
return { facets: [] };
|
|
3352
|
+
}
|
|
3346
3353
|
}
|
|
3347
3354
|
// Merge with live values
|
|
3348
3355
|
const allValues = valuesManager.getAll();
|
|
@@ -3820,35 +3827,6 @@ function createSearchHandler(deps) {
|
|
|
3820
3827
|
}, deps.logger, 'Search');
|
|
3821
3828
|
}
|
|
3822
3829
|
|
|
3823
|
-
/**
|
|
3824
|
-
* @module api/handlers/status
|
|
3825
|
-
* Fastify route handler for GET /status. Returns process health, uptime, and collection stats.
|
|
3826
|
-
*/
|
|
3827
|
-
/**
|
|
3828
|
-
* Create handler for GET /status.
|
|
3829
|
-
*
|
|
3830
|
-
* @param deps - Route dependencies.
|
|
3831
|
-
*/
|
|
3832
|
-
function createStatusHandler(deps) {
|
|
3833
|
-
return async () => {
|
|
3834
|
-
const collectionInfo = await deps.vectorStore.getCollectionInfo();
|
|
3835
|
-
return {
|
|
3836
|
-
status: 'ok',
|
|
3837
|
-
version: deps.version,
|
|
3838
|
-
uptime: process.uptime(),
|
|
3839
|
-
collection: {
|
|
3840
|
-
name: deps.getCollectionName(),
|
|
3841
|
-
pointCount: collectionInfo.pointCount,
|
|
3842
|
-
dimensions: collectionInfo.dimensions,
|
|
3843
|
-
},
|
|
3844
|
-
reindex: deps.reindexTracker.getStatus(),
|
|
3845
|
-
...(deps.initialScanTracker
|
|
3846
|
-
? { initialScan: deps.initialScanTracker.getStatus() }
|
|
3847
|
-
: {}),
|
|
3848
|
-
};
|
|
3849
|
-
};
|
|
3850
|
-
}
|
|
3851
|
-
|
|
3852
3830
|
/**
|
|
3853
3831
|
* @module api/handlers/walk
|
|
3854
3832
|
* Fastify route handler for POST /walk. Enumerates watched files matching caller-provided globs
|
|
@@ -4314,8 +4292,12 @@ function createOnRulesChanged(deps) {
|
|
|
4314
4292
|
const configCompiled = compileRules(configRules);
|
|
4315
4293
|
const virtualCompiled = virtualRuleStore.getCompiled();
|
|
4316
4294
|
// Rebuild template engine asynchronously with merged rules
|
|
4317
|
-
|
|
4295
|
+
buildTemplateEngineAndCustomMapLib({ ...config, inferenceRules: mergedRules }, dirname(configPath))
|
|
4296
|
+
.then(({ templateEngine: newEngine, customMapLib: newMapLib }) => {
|
|
4318
4297
|
processor.updateRules([...configCompiled, ...virtualCompiled], newEngine, newMapLib);
|
|
4298
|
+
})
|
|
4299
|
+
.catch((err) => {
|
|
4300
|
+
logger.error({ err }, 'onRulesChanged: failed to rebuild template engine; processor rules not updated');
|
|
4319
4301
|
});
|
|
4320
4302
|
// Auto-trigger rules reindex scoped to newly registered rule globs (Fix 21)
|
|
4321
4303
|
const matchGlobs = extractMatchGlobs(allVirtualRules);
|
|
@@ -4485,13 +4467,28 @@ function createApiServer(options) {
|
|
|
4485
4467
|
}, scope);
|
|
4486
4468
|
};
|
|
4487
4469
|
const cacheTtlMs = config.api?.cacheTtlMs ?? 30000;
|
|
4488
|
-
|
|
4489
|
-
|
|
4490
|
-
getCollectionName: () => getConfig().vectorStore.collectionName,
|
|
4491
|
-
reindexTracker,
|
|
4470
|
+
const coreStatusHandler = createStatusHandler({
|
|
4471
|
+
name: 'watcher',
|
|
4492
4472
|
version: version ?? 'unknown',
|
|
4493
|
-
|
|
4494
|
-
|
|
4473
|
+
getHealth: async () => {
|
|
4474
|
+
const collectionInfo = await vectorStore.getCollectionInfo();
|
|
4475
|
+
return {
|
|
4476
|
+
collection: {
|
|
4477
|
+
name: getConfig().vectorStore.collectionName,
|
|
4478
|
+
pointCount: collectionInfo.pointCount,
|
|
4479
|
+
dimensions: collectionInfo.dimensions,
|
|
4480
|
+
},
|
|
4481
|
+
reindex: reindexTracker.getStatus(),
|
|
4482
|
+
...(initialScanTracker
|
|
4483
|
+
? { initialScan: initialScanTracker.getStatus() }
|
|
4484
|
+
: {}),
|
|
4485
|
+
};
|
|
4486
|
+
},
|
|
4487
|
+
});
|
|
4488
|
+
app.get('/status', withCache(cacheTtlMs, async () => {
|
|
4489
|
+
const result = await coreStatusHandler();
|
|
4490
|
+
return result.body;
|
|
4491
|
+
}));
|
|
4495
4492
|
app.post('/metadata', createMetadataHandler({
|
|
4496
4493
|
processor,
|
|
4497
4494
|
getConfig,
|
|
@@ -5240,6 +5237,27 @@ const EMBEDDING_DEFAULTS = {
|
|
|
5240
5237
|
rateLimitPerMinute: 300,
|
|
5241
5238
|
concurrency: 5,
|
|
5242
5239
|
};
|
|
5240
|
+
/** Default init command config template. */
|
|
5241
|
+
const INIT_CONFIG_TEMPLATE = {
|
|
5242
|
+
$schema: 'node_modules/@karmaniverous/jeeves-watcher/config.schema.json',
|
|
5243
|
+
watch: {
|
|
5244
|
+
paths: ['**/*.{md,markdown,txt,text,json,html,htm,pdf,docx}'],
|
|
5245
|
+
ignored: ['**/node_modules/**', '**/.git/**', '**/.jeeves-watcher/**'],
|
|
5246
|
+
},
|
|
5247
|
+
configWatch: CONFIG_WATCH_DEFAULTS,
|
|
5248
|
+
embedding: {
|
|
5249
|
+
provider: 'gemini',
|
|
5250
|
+
model: 'gemini-embedding-001',
|
|
5251
|
+
dimensions: EMBEDDING_DEFAULTS.dimensions,
|
|
5252
|
+
},
|
|
5253
|
+
vectorStore: {
|
|
5254
|
+
url: 'http://127.0.0.1:6333',
|
|
5255
|
+
collectionName: 'jeeves-watcher',
|
|
5256
|
+
},
|
|
5257
|
+
stateDir: ROOT_DEFAULTS.stateDir,
|
|
5258
|
+
api: API_DEFAULTS,
|
|
5259
|
+
logging: LOGGING_DEFAULTS,
|
|
5260
|
+
};
|
|
5243
5261
|
|
|
5244
5262
|
/**
|
|
5245
5263
|
* @module config/substituteEnvVars
|
|
@@ -5353,7 +5371,9 @@ async function loadConfig(configPath) {
|
|
|
5353
5371
|
const errors = error.issues
|
|
5354
5372
|
.map((issue) => `${issue.path.join('.')}: ${issue.message}`)
|
|
5355
5373
|
.join('; ');
|
|
5356
|
-
throw new Error(`Invalid jeeves-watcher configuration: ${errors}
|
|
5374
|
+
throw new Error(`Invalid jeeves-watcher configuration: ${errors}`, {
|
|
5375
|
+
cause: error,
|
|
5376
|
+
});
|
|
5357
5377
|
}
|
|
5358
5378
|
throw error;
|
|
5359
5379
|
}
|
|
@@ -5894,6 +5914,8 @@ function computeLineOffsets(fullText, chunks) {
|
|
|
5894
5914
|
* @module processor/processingPipeline
|
|
5895
5915
|
* Extracted embed→chunk→upsert pipeline for DocumentProcessor.
|
|
5896
5916
|
*/
|
|
5917
|
+
/** Default number of points to upsert per batch (prevents OOM on large files). */
|
|
5918
|
+
const DEFAULT_UPSERT_BATCH_SIZE = 50;
|
|
5897
5919
|
/**
|
|
5898
5920
|
* Embed text chunks, upsert to vector store, and clean up orphaned chunks.
|
|
5899
5921
|
*
|
|
@@ -5905,7 +5927,7 @@ function computeLineOffsets(fullText, chunks) {
|
|
|
5905
5927
|
* @param fileDates - File creation and modification timestamps (unix seconds).
|
|
5906
5928
|
*/
|
|
5907
5929
|
async function embedAndUpsert(deps, text, filePath, metadata, existingPayload, fileDates) {
|
|
5908
|
-
const { embeddingProvider, vectorStore, splitter, logger } = deps;
|
|
5930
|
+
const { embeddingProvider, vectorStore, splitter, logger, upsertBatchSize = DEFAULT_UPSERT_BATCH_SIZE, } = deps;
|
|
5909
5931
|
const oldTotalChunks = getChunkCount(existingPayload);
|
|
5910
5932
|
const hash = contentHash(text);
|
|
5911
5933
|
// Chunk text
|
|
@@ -5914,7 +5936,7 @@ async function embedAndUpsert(deps, text, filePath, metadata, existingPayload, f
|
|
|
5914
5936
|
const offsets = computeLineOffsets(text, chunks);
|
|
5915
5937
|
// Embed all chunks
|
|
5916
5938
|
const vectors = await embeddingProvider.embed(chunks);
|
|
5917
|
-
//
|
|
5939
|
+
// Build all points
|
|
5918
5940
|
const points = chunks.map((chunk, i) => ({
|
|
5919
5941
|
id: pointId(filePath, i),
|
|
5920
5942
|
vector: vectors[i],
|
|
@@ -5931,7 +5953,11 @@ async function embedAndUpsert(deps, text, filePath, metadata, existingPayload, f
|
|
|
5931
5953
|
[FIELD_LINE_END]: offsets[i]?.lineEnd ?? 1,
|
|
5932
5954
|
},
|
|
5933
5955
|
}));
|
|
5934
|
-
|
|
5956
|
+
// Upsert in batches to avoid OOM on large files (#162)
|
|
5957
|
+
const batchSize = Math.max(1, upsertBatchSize);
|
|
5958
|
+
for (let start = 0; start < points.length; start += batchSize) {
|
|
5959
|
+
await vectorStore.upsert(points.slice(start, start + batchSize));
|
|
5960
|
+
}
|
|
5935
5961
|
// Clean up orphaned chunks
|
|
5936
5962
|
if (oldTotalChunks > chunks.length) {
|
|
5937
5963
|
const orphanIds = chunkIds(filePath, oldTotalChunks).slice(chunks.length);
|
|
@@ -6261,6 +6287,7 @@ class EventQueue {
|
|
|
6261
6287
|
tokens;
|
|
6262
6288
|
lastRefillMs = Date.now();
|
|
6263
6289
|
drainWaiters = [];
|
|
6290
|
+
idleWorkerWaiters = [];
|
|
6264
6291
|
/**
|
|
6265
6292
|
* Create an event queue.
|
|
6266
6293
|
*
|
|
@@ -6327,6 +6354,20 @@ class EventQueue {
|
|
|
6327
6354
|
this.drainWaiters.push(resolve);
|
|
6328
6355
|
});
|
|
6329
6356
|
}
|
|
6357
|
+
/**
|
|
6358
|
+
* Wait until there is no in-flight work.
|
|
6359
|
+
*
|
|
6360
|
+
* Unlike {@link drain}, this does NOT wait for debounced or queued items to be empty.
|
|
6361
|
+
* It's useful when you need to temporarily quiesce outgoing work without losing
|
|
6362
|
+
* or rejecting new enqueues.
|
|
6363
|
+
*/
|
|
6364
|
+
async waitForIdleWorkers() {
|
|
6365
|
+
if (this.active === 0)
|
|
6366
|
+
return;
|
|
6367
|
+
await new Promise((resolve) => {
|
|
6368
|
+
this.idleWorkerWaiters.push(resolve);
|
|
6369
|
+
});
|
|
6370
|
+
}
|
|
6330
6371
|
push(item) {
|
|
6331
6372
|
if (item.event.priority === 'low')
|
|
6332
6373
|
this.lowQueue.push(item);
|
|
@@ -6378,6 +6419,7 @@ class EventQueue {
|
|
|
6378
6419
|
this.active -= 1;
|
|
6379
6420
|
this.pump();
|
|
6380
6421
|
this.maybeResolveDrain();
|
|
6422
|
+
this.maybeResolveIdleWorkers();
|
|
6381
6423
|
});
|
|
6382
6424
|
}
|
|
6383
6425
|
this.maybeResolveDrain();
|
|
@@ -6397,6 +6439,14 @@ class EventQueue {
|
|
|
6397
6439
|
for (const resolve of waiters)
|
|
6398
6440
|
resolve();
|
|
6399
6441
|
}
|
|
6442
|
+
maybeResolveIdleWorkers() {
|
|
6443
|
+
if (this.active !== 0)
|
|
6444
|
+
return;
|
|
6445
|
+
const waiters = this.idleWorkerWaiters;
|
|
6446
|
+
this.idleWorkerWaiters = [];
|
|
6447
|
+
for (const resolve of waiters)
|
|
6448
|
+
resolve();
|
|
6449
|
+
}
|
|
6400
6450
|
}
|
|
6401
6451
|
|
|
6402
6452
|
/**
|
|
@@ -6546,7 +6596,7 @@ async function ensureTextIndex(client, collectionName, fieldName, log) {
|
|
|
6546
6596
|
log.info({ fieldName }, 'Full-text payload index created');
|
|
6547
6597
|
}
|
|
6548
6598
|
catch (error) {
|
|
6549
|
-
throw new Error(`Failed to create text index on "${fieldName}": ${String(error)}
|
|
6599
|
+
throw new Error(`Failed to create text index on "${fieldName}": ${String(error)}`, { cause: error });
|
|
6550
6600
|
}
|
|
6551
6601
|
}
|
|
6552
6602
|
/**
|
|
@@ -6726,7 +6776,7 @@ class VectorStoreClient {
|
|
|
6726
6776
|
}
|
|
6727
6777
|
}
|
|
6728
6778
|
catch (error) {
|
|
6729
|
-
throw new Error(`Failed to ensure collection "${this.collectionName}": ${String(error)}
|
|
6779
|
+
throw new Error(`Failed to ensure collection "${this.collectionName}": ${String(error)}`, { cause: error });
|
|
6730
6780
|
}
|
|
6731
6781
|
}
|
|
6732
6782
|
/**
|
|
@@ -7571,6 +7621,53 @@ const defaultFactories = {
|
|
|
7571
7621
|
createApiServer,
|
|
7572
7622
|
};
|
|
7573
7623
|
|
|
7624
|
+
/**
|
|
7625
|
+
* @module config/migrateConfigPath
|
|
7626
|
+
*
|
|
7627
|
+
* Auto-migrates legacy flat config path (`jeeves-watcher.config.json`)
|
|
7628
|
+
* to the new namespaced convention (`jeeves-watcher/config.json`).
|
|
7629
|
+
*/
|
|
7630
|
+
const OLD_FILENAME = 'jeeves-watcher.config.json';
|
|
7631
|
+
const NEW_DIR = 'jeeves-watcher';
|
|
7632
|
+
const NEW_FILENAME = 'config.json';
|
|
7633
|
+
/**
|
|
7634
|
+
* Resolve the config path, migrating from old to new convention if needed.
|
|
7635
|
+
*
|
|
7636
|
+
* When `configDir` is provided (the directory to search for config), the
|
|
7637
|
+
* function checks for the old flat file and the new namespaced directory.
|
|
7638
|
+
*
|
|
7639
|
+
* Migration only runs when using the default/conventional path — an explicit
|
|
7640
|
+
* `-c` flag bypasses this entirely.
|
|
7641
|
+
*
|
|
7642
|
+
* @param configDir - The directory to search for config files.
|
|
7643
|
+
* @returns The resolved config path and migration metadata.
|
|
7644
|
+
* @throws If neither old nor new config path exists.
|
|
7645
|
+
*/
|
|
7646
|
+
function migrateConfigPath(configDir) {
|
|
7647
|
+
const oldPath = join(configDir, OLD_FILENAME);
|
|
7648
|
+
const newDir = join(configDir, NEW_DIR);
|
|
7649
|
+
const newPath = join(newDir, NEW_FILENAME);
|
|
7650
|
+
const oldExists = existsSync(oldPath);
|
|
7651
|
+
const newExists = existsSync(newPath);
|
|
7652
|
+
if (newExists && oldExists) {
|
|
7653
|
+
return {
|
|
7654
|
+
configPath: newPath,
|
|
7655
|
+
migrated: false,
|
|
7656
|
+
warning: `Both legacy config (${oldPath}) and new config (${newPath}) exist. Using new path. Consider removing the legacy file.`,
|
|
7657
|
+
};
|
|
7658
|
+
}
|
|
7659
|
+
if (newExists) {
|
|
7660
|
+
return { configPath: newPath, migrated: false };
|
|
7661
|
+
}
|
|
7662
|
+
if (oldExists) {
|
|
7663
|
+
mkdirSync(newDir, { recursive: true });
|
|
7664
|
+
renameSync(oldPath, newPath);
|
|
7665
|
+
return { configPath: newPath, migrated: true };
|
|
7666
|
+
}
|
|
7667
|
+
throw new Error(`No jeeves-watcher configuration found in ${configDir}. ` +
|
|
7668
|
+
`Expected ${newPath} or ${oldPath}.`);
|
|
7669
|
+
}
|
|
7670
|
+
|
|
7574
7671
|
/**
|
|
7575
7672
|
* @module app/shutdown
|
|
7576
7673
|
* Process signal shutdown orchestration. Installs SIGINT/SIGTERM handlers that invoke a provided async stop function.
|
|
@@ -7596,12 +7693,38 @@ function installShutdownHandlers(stop) {
|
|
|
7596
7693
|
/**
|
|
7597
7694
|
* Create and start a JeevesWatcher from a config file path.
|
|
7598
7695
|
*
|
|
7599
|
-
*
|
|
7696
|
+
* When no explicit config path is given, auto-migrates the legacy flat
|
|
7697
|
+
* config (`jeeves-watcher.config.json`) to the namespaced convention
|
|
7698
|
+
* (`jeeves-watcher/config.json`) before loading.
|
|
7699
|
+
*
|
|
7700
|
+
* @param configPath - Optional explicit path to the configuration file.
|
|
7701
|
+
* When provided, the file is loaded as-is (no migration).
|
|
7600
7702
|
* @returns The running JeevesWatcher instance.
|
|
7601
7703
|
*/
|
|
7602
7704
|
async function startFromConfig(configPath) {
|
|
7603
|
-
|
|
7604
|
-
|
|
7705
|
+
let resolvedPath = configPath;
|
|
7706
|
+
// Auto-migrate only when no explicit path was given.
|
|
7707
|
+
if (!configPath) {
|
|
7708
|
+
try {
|
|
7709
|
+
const result = migrateConfigPath(process.cwd());
|
|
7710
|
+
resolvedPath = result.configPath;
|
|
7711
|
+
if (result.migrated) {
|
|
7712
|
+
console.log(`[jeeves-watcher] Migrated config to ${result.configPath}`);
|
|
7713
|
+
}
|
|
7714
|
+
if (result.warning) {
|
|
7715
|
+
console.warn(`[jeeves-watcher] ${result.warning}`);
|
|
7716
|
+
}
|
|
7717
|
+
}
|
|
7718
|
+
catch (error) {
|
|
7719
|
+
// Migration discovery failed — log and fall through to loadConfig
|
|
7720
|
+
// which will use cosmiconfig's own search.
|
|
7721
|
+
console.warn('[jeeves-watcher] Config migration check failed: ' +
|
|
7722
|
+
(error instanceof Error ? error.message : String(error)));
|
|
7723
|
+
resolvedPath = undefined;
|
|
7724
|
+
}
|
|
7725
|
+
}
|
|
7726
|
+
const config = await loadConfig(resolvedPath);
|
|
7727
|
+
const app = new JeevesWatcher(config, resolvedPath);
|
|
7605
7728
|
installShutdownHandlers(() => app.stop());
|
|
7606
7729
|
await app.start();
|
|
7607
7730
|
return app;
|
|
@@ -7749,7 +7872,7 @@ class JeevesWatcher {
|
|
|
7749
7872
|
enrichmentStore: this.enrichmentStore,
|
|
7750
7873
|
});
|
|
7751
7874
|
await server.listen({
|
|
7752
|
-
host: this.config.api?.host ?? '
|
|
7875
|
+
host: this.config.api?.host ?? getBindAddress('watcher'),
|
|
7753
7876
|
port: this.config.api?.port ?? 1936,
|
|
7754
7877
|
});
|
|
7755
7878
|
return server;
|
|
@@ -7808,4 +7931,275 @@ class JeevesWatcher {
|
|
|
7808
7931
|
}
|
|
7809
7932
|
}
|
|
7810
7933
|
|
|
7811
|
-
|
|
7934
|
+
/**
|
|
7935
|
+
* @module cli/jeeves-watcher/customCommands
|
|
7936
|
+
* Domain-specific CLI commands registered via the descriptor's customCliCommands.
|
|
7937
|
+
* Uses fetchJson/postJson from core instead of hand-rolled API helpers.
|
|
7938
|
+
*/
|
|
7939
|
+
const DEFAULT_PORT = '1936';
|
|
7940
|
+
const DEFAULT_HOST = '127.0.0.1';
|
|
7941
|
+
/** Build the API base URL from host and port options. */
|
|
7942
|
+
function baseUrl(opts) {
|
|
7943
|
+
return `http://${opts.host}:${opts.port}`;
|
|
7944
|
+
}
|
|
7945
|
+
/** Add standard --port and --host options to a command. */
|
|
7946
|
+
function withApiOptions(cmd) {
|
|
7947
|
+
return cmd
|
|
7948
|
+
.option('-p, --port <port>', 'API port', DEFAULT_PORT)
|
|
7949
|
+
.option('-H, --host <host>', 'API host', DEFAULT_HOST);
|
|
7950
|
+
}
|
|
7951
|
+
/** Wrap an async action with standard error handling. */
|
|
7952
|
+
function handleErrors(fn) {
|
|
7953
|
+
return async () => {
|
|
7954
|
+
try {
|
|
7955
|
+
await fn();
|
|
7956
|
+
}
|
|
7957
|
+
catch (err) {
|
|
7958
|
+
console.error(err instanceof Error ? err.message : String(err));
|
|
7959
|
+
process.exitCode = 1;
|
|
7960
|
+
}
|
|
7961
|
+
};
|
|
7962
|
+
}
|
|
7963
|
+
/** Print JSON response to stdout. */
|
|
7964
|
+
function printJson(data) {
|
|
7965
|
+
console.log(JSON.stringify(data, null, 2));
|
|
7966
|
+
}
|
|
7967
|
+
/**
|
|
7968
|
+
* Register all domain-specific commands on the given Commander program.
|
|
7969
|
+
*
|
|
7970
|
+
* @param program - The Commander program from createServiceCli.
|
|
7971
|
+
*/
|
|
7972
|
+
function registerCustomCommands(program) {
|
|
7973
|
+
// --- search ---
|
|
7974
|
+
withApiOptions(program
|
|
7975
|
+
.command('search')
|
|
7976
|
+
.description('Search the vector store (POST /search)')
|
|
7977
|
+
.argument('<query>', 'Search query')
|
|
7978
|
+
.option('-l, --limit <limit>', 'Max results', '10')).action(async (query, opts) => {
|
|
7979
|
+
await handleErrors(async () => {
|
|
7980
|
+
const result = await postJson(`${baseUrl(opts)}/search`, {
|
|
7981
|
+
query,
|
|
7982
|
+
limit: Number(opts.limit),
|
|
7983
|
+
});
|
|
7984
|
+
printJson(result);
|
|
7985
|
+
})();
|
|
7986
|
+
});
|
|
7987
|
+
// --- enrich ---
|
|
7988
|
+
withApiOptions(program
|
|
7989
|
+
.command('enrich')
|
|
7990
|
+
.description('Enrich document metadata (POST /metadata)')
|
|
7991
|
+
.argument('<path>', 'File path to enrich')
|
|
7992
|
+
.option('-k, --key <key=value...>', 'Metadata key-value pairs (repeatable)', [])
|
|
7993
|
+
.option('-j, --json <json>', 'Metadata as JSON string (e.g., \'{"key":"value"}\')')).action(async (path, opts) => {
|
|
7994
|
+
await handleErrors(async () => {
|
|
7995
|
+
const metadata = parseMetadataArgs(opts.json, opts.key);
|
|
7996
|
+
if (!metadata)
|
|
7997
|
+
return;
|
|
7998
|
+
const result = await postJson(`${baseUrl(opts)}/metadata`, {
|
|
7999
|
+
path,
|
|
8000
|
+
metadata,
|
|
8001
|
+
});
|
|
8002
|
+
printJson(result);
|
|
8003
|
+
})();
|
|
8004
|
+
});
|
|
8005
|
+
// --- scan ---
|
|
8006
|
+
withApiOptions(program
|
|
8007
|
+
.command('scan')
|
|
8008
|
+
.description('Scan the vector store (POST /scan)')
|
|
8009
|
+
.option('-f, --filter <filter>', 'Qdrant filter (JSON string)', '{}')
|
|
8010
|
+
.option('-l, --limit <limit>', 'Max results', '100')
|
|
8011
|
+
.option('-c, --cursor <cursor>', 'Cursor from previous response')
|
|
8012
|
+
.option('--fields <fields>', 'Fields to return (comma-separated)')
|
|
8013
|
+
.option('--count-only', 'Return count only')).action(async (opts) => {
|
|
8014
|
+
await handleErrors(async () => {
|
|
8015
|
+
const filterObj = JSON.parse(opts.filter);
|
|
8016
|
+
const result = await postJson(`${baseUrl(opts)}/scan`, {
|
|
8017
|
+
filter: filterObj,
|
|
8018
|
+
limit: Number(opts.limit),
|
|
8019
|
+
cursor: opts.cursor,
|
|
8020
|
+
fields: opts.fields
|
|
8021
|
+
? opts.fields.split(',').map((f) => f.trim())
|
|
8022
|
+
: undefined,
|
|
8023
|
+
countOnly: opts.countOnly,
|
|
8024
|
+
});
|
|
8025
|
+
printJson(result);
|
|
8026
|
+
})();
|
|
8027
|
+
});
|
|
8028
|
+
// --- reindex ---
|
|
8029
|
+
withApiOptions(program
|
|
8030
|
+
.command('reindex')
|
|
8031
|
+
.description('Trigger a reindex operation (POST /reindex)')
|
|
8032
|
+
.option('-s, --scope <scope>', 'Reindex scope (issues|full|rules|path|prune)', 'rules')
|
|
8033
|
+
.option('-t, --path <paths...>', 'Target path(s) for path or rules scope')).action(async (opts) => {
|
|
8034
|
+
await handleErrors(async () => {
|
|
8035
|
+
if (!VALID_SCOPES.includes(opts.scope)) {
|
|
8036
|
+
console.error(`Invalid scope "${opts.scope}". Must be one of: ${VALID_SCOPES.join(', ')}`);
|
|
8037
|
+
process.exitCode = 1;
|
|
8038
|
+
return;
|
|
8039
|
+
}
|
|
8040
|
+
const body = { scope: opts.scope };
|
|
8041
|
+
if (opts.path) {
|
|
8042
|
+
body.path = opts.path.length === 1 ? opts.path[0] : opts.path;
|
|
8043
|
+
}
|
|
8044
|
+
const result = await postJson(`${baseUrl(opts)}/reindex`, body);
|
|
8045
|
+
printJson(result);
|
|
8046
|
+
})();
|
|
8047
|
+
});
|
|
8048
|
+
// --- rebuild-metadata ---
|
|
8049
|
+
withApiOptions(program
|
|
8050
|
+
.command('rebuild-metadata')
|
|
8051
|
+
.description('Rebuild metadata store from Qdrant (POST /rebuild-metadata)')).action(async (opts) => {
|
|
8052
|
+
await handleErrors(async () => {
|
|
8053
|
+
const result = await postJson(`${baseUrl(opts)}/rebuild-metadata`, {});
|
|
8054
|
+
printJson(result);
|
|
8055
|
+
})();
|
|
8056
|
+
});
|
|
8057
|
+
// --- issues ---
|
|
8058
|
+
withApiOptions(program
|
|
8059
|
+
.command('issues')
|
|
8060
|
+
.description('List current processing issues (GET /issues)')).action(async (opts) => {
|
|
8061
|
+
await handleErrors(async () => {
|
|
8062
|
+
const result = await fetchJson(`${baseUrl(opts)}/issues`);
|
|
8063
|
+
printJson(result);
|
|
8064
|
+
})();
|
|
8065
|
+
});
|
|
8066
|
+
// --- helpers ---
|
|
8067
|
+
withApiOptions(program.command('helpers').description('List available helper functions')).action(async (opts) => {
|
|
8068
|
+
await handleErrors(async () => {
|
|
8069
|
+
const base = baseUrl(opts);
|
|
8070
|
+
const mapPath = encodeURIComponent('$.mapHelpers');
|
|
8071
|
+
const tplPath = encodeURIComponent('$.templateHelpers');
|
|
8072
|
+
const [mapResult, tplResult] = await Promise.all([
|
|
8073
|
+
fetchJson(`${base}/config?path=${mapPath}`),
|
|
8074
|
+
fetchJson(`${base}/config?path=${tplPath}`),
|
|
8075
|
+
]);
|
|
8076
|
+
const mapData = mapResult.result[0];
|
|
8077
|
+
const tplData = tplResult.result[0];
|
|
8078
|
+
const sections = [];
|
|
8079
|
+
const mapSection = formatHelperSection('JsonMap lib functions', mapData);
|
|
8080
|
+
if (mapSection)
|
|
8081
|
+
sections.push(mapSection);
|
|
8082
|
+
const tplSection = formatHelperSection('Handlebars helpers', tplData);
|
|
8083
|
+
if (tplSection)
|
|
8084
|
+
sections.push(tplSection);
|
|
8085
|
+
if (sections.length === 0) {
|
|
8086
|
+
console.log('No helpers configured.');
|
|
8087
|
+
}
|
|
8088
|
+
else {
|
|
8089
|
+
console.log(sections.join('\n\n'));
|
|
8090
|
+
}
|
|
8091
|
+
})();
|
|
8092
|
+
});
|
|
8093
|
+
}
|
|
8094
|
+
// --- shared constants and helpers ---
|
|
8095
|
+
const VALID_SCOPES = ['issues', 'full', 'rules', 'path', 'prune'];
|
|
8096
|
+
/** Parse --json and --key options into a metadata object. Returns null on validation failure. */
|
|
8097
|
+
function parseMetadataArgs(json, keys) {
|
|
8098
|
+
let metadata = {};
|
|
8099
|
+
if (json) {
|
|
8100
|
+
try {
|
|
8101
|
+
metadata = JSON.parse(json);
|
|
8102
|
+
}
|
|
8103
|
+
catch {
|
|
8104
|
+
console.error('Invalid JSON:', json);
|
|
8105
|
+
process.exitCode = 1;
|
|
8106
|
+
return null;
|
|
8107
|
+
}
|
|
8108
|
+
}
|
|
8109
|
+
if (Array.isArray(keys) && keys.length > 0) {
|
|
8110
|
+
for (const pair of keys) {
|
|
8111
|
+
const eqIndex = pair.indexOf('=');
|
|
8112
|
+
if (eqIndex === -1) {
|
|
8113
|
+
console.error(`Invalid key-value pair: ${pair}`);
|
|
8114
|
+
process.exitCode = 1;
|
|
8115
|
+
return null;
|
|
8116
|
+
}
|
|
8117
|
+
metadata[pair.slice(0, eqIndex)] = pair.slice(eqIndex + 1);
|
|
8118
|
+
}
|
|
8119
|
+
}
|
|
8120
|
+
if (Object.keys(metadata).length === 0) {
|
|
8121
|
+
console.error('No metadata provided. Use --key or --json.');
|
|
8122
|
+
process.exitCode = 1;
|
|
8123
|
+
return null;
|
|
8124
|
+
}
|
|
8125
|
+
return metadata;
|
|
8126
|
+
}
|
|
8127
|
+
/** Format a helpers section for display. */
|
|
8128
|
+
function formatHelperSection(title, data) {
|
|
8129
|
+
if (!data || Object.keys(data).length === 0)
|
|
8130
|
+
return '';
|
|
8131
|
+
const lines = [`${title}:`];
|
|
8132
|
+
for (const [namespace, entry] of Object.entries(data)) {
|
|
8133
|
+
if (entry.description) {
|
|
8134
|
+
lines.push(` [${namespace}] ${entry.description}`);
|
|
8135
|
+
}
|
|
8136
|
+
if (entry.exports) {
|
|
8137
|
+
for (const [name, desc] of Object.entries(entry.exports)) {
|
|
8138
|
+
const descPart = desc ? ` - ${desc}` : '';
|
|
8139
|
+
lines.push(` ${name.padEnd(40)}${descPart}`);
|
|
8140
|
+
}
|
|
8141
|
+
}
|
|
8142
|
+
else {
|
|
8143
|
+
lines.push(` (no exports introspected)`);
|
|
8144
|
+
}
|
|
8145
|
+
}
|
|
8146
|
+
return lines.join('\n');
|
|
8147
|
+
}
|
|
8148
|
+
|
|
8149
|
+
/**
|
|
8150
|
+
* @module descriptor
|
|
8151
|
+
* Jeeves Component Descriptor for the watcher service. Single source of truth
|
|
8152
|
+
* consumed by core factories (CLI, plugin tools, HTTP handlers, service manager).
|
|
8153
|
+
*/
|
|
8154
|
+
const require$1 = createRequire(import.meta.url);
|
|
8155
|
+
const { version } = require$1('../package.json');
|
|
8156
|
+
/**
|
|
8157
|
+
* Watcher component descriptor. Single source of truth for service identity,
|
|
8158
|
+
* config schema, and extension points consumed by core factories.
|
|
8159
|
+
*/
|
|
8160
|
+
const watcherDescriptor = {
|
|
8161
|
+
// Identity
|
|
8162
|
+
name: 'watcher',
|
|
8163
|
+
version,
|
|
8164
|
+
servicePackage: '@karmaniverous/jeeves-watcher',
|
|
8165
|
+
pluginPackage: '@karmaniverous/jeeves-watcher-openclaw',
|
|
8166
|
+
defaultPort: 1936,
|
|
8167
|
+
// Config
|
|
8168
|
+
configSchema: jeevesWatcherConfigSchema,
|
|
8169
|
+
configFileName: 'config.json',
|
|
8170
|
+
initTemplate: () => ({ ...INIT_CONFIG_TEMPLATE }),
|
|
8171
|
+
// Service behavior — onConfigApply wired at the Fastify layer (api/index.ts)
|
|
8172
|
+
// where it has access to the live reindex tracker and config getter.
|
|
8173
|
+
onConfigApply: async (config) => {
|
|
8174
|
+
await Promise.resolve();
|
|
8175
|
+
},
|
|
8176
|
+
customMerge: (target, source) => {
|
|
8177
|
+
const mergedRules = mergeInferenceRules(target['inferenceRules'], source['inferenceRules']);
|
|
8178
|
+
return {
|
|
8179
|
+
...target,
|
|
8180
|
+
...source,
|
|
8181
|
+
inferenceRules: mergedRules,
|
|
8182
|
+
};
|
|
8183
|
+
},
|
|
8184
|
+
startCommand: (configPath) => [
|
|
8185
|
+
'node',
|
|
8186
|
+
fileURLToPath(new URL('./cli/jeeves-watcher/index.js', import.meta.url)),
|
|
8187
|
+
'start',
|
|
8188
|
+
'-c',
|
|
8189
|
+
configPath,
|
|
8190
|
+
],
|
|
8191
|
+
// Content — generateToolsContent is wired in the plugin package
|
|
8192
|
+
// (watcherComponent.ts) where it has access to the API URL for menu generation.
|
|
8193
|
+
sectionId: 'Watcher',
|
|
8194
|
+
refreshIntervalSeconds: 71,
|
|
8195
|
+
generateToolsContent: () => '',
|
|
8196
|
+
// Extension points
|
|
8197
|
+
customCliCommands: (program) => {
|
|
8198
|
+
registerCustomCommands(program);
|
|
8199
|
+
},
|
|
8200
|
+
customPluginTools: (api) => {
|
|
8201
|
+
return [];
|
|
8202
|
+
},
|
|
8203
|
+
};
|
|
8204
|
+
|
|
8205
|
+
export { ContentHashCache, DocumentProcessor, EnrichmentStore, EventQueue, FileSystemWatcher, GitignoreFilter, InitialScanTracker, IssuesManager, JeevesWatcher, ReindexTracker, SystemHealth, TemplateEngine, ValuesManager, VectorStoreClient, VirtualRuleStore, apiConfigSchema, applyRules, buildAttributes, buildTemplateEngine, compileRules, configWatchConfigSchema, contentHash, createApiServer, createEmbeddingProvider, createHandlebarsInstance, createLogger, embeddingConfigSchema, extractText, inferenceRuleSchema, issueRecordSchema, jeevesWatcherConfigSchema, loadConfig, loadCustomHelpers, loggingConfigSchema, mergeEnrichment, pointId, registerBuiltinHelpers, resolveTemplateSource, startFromConfig, vectorStoreConfigSchema, watchConfigSchema, watcherDescriptor };
|