@karmaniverous/jeeves-watcher 0.19.1 → 0.20.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +28 -0
- package/README.md +22 -16
- package/config.schema.json +267 -115
- package/dist/cli/jeeves-watcher/{index-C_bf9c9m.js → index-C9-xTM7k.js} +7720 -3114
- package/dist/cli/jeeves-watcher/index.js +773 -1147
- package/dist/index.d.ts +713 -209
- package/dist/index.js +10169 -5871
- package/package.json +22 -20
|
@@ -1,32 +1,17 @@
|
|
|
1
1
|
#!/usr/bin/env node
|
|
2
|
-
import { substituteEnvVars,
|
|
3
|
-
import {
|
|
2
|
+
import { substituteEnvVars, getPackageVersion, createServiceCli } from '@karmaniverous/jeeves';
|
|
3
|
+
import { resolve, dirname, join } from 'node:path';
|
|
4
4
|
import { fileURLToPath } from 'node:url';
|
|
5
|
-
import { jeevesWatcherConfigSchema, LOGGING_DEFAULTS, API_DEFAULTS, EMBEDDING_DEFAULTS, CONFIG_WATCH_DEFAULTS, WATCH_DEFAULTS, ROOT_DEFAULTS,
|
|
5
|
+
import { isPlainObject as isPlainObject$1, findReservedPropertyNames, sourceConfigSchema, jeevesWatcherConfigSchema, LOGGING_DEFAULTS, API_DEFAULTS, EMBEDDING_DEFAULTS, CONFIG_WATCH_DEFAULTS, WATCH_DEFAULTS, ROOT_DEFAULTS, bearerHeaders, API_KEY_ENV_VAR, httpError, formatHttpError, unauthorizedMessage, getEndpoint, reindexScopeSchema, INIT_CONFIG_TEMPLATE, DEFAULT_PORT as DEFAULT_PORT$1, PLUGIN_PACKAGE, SERVICE_PACKAGE, COMPONENT_NAME } from '@karmaniverous/jeeves-watcher-core';
|
|
6
6
|
import { packageDirectorySync } from 'package-directory';
|
|
7
|
-
import { readFileSync, existsSync, mkdirSync, renameSync
|
|
7
|
+
import { readFileSync, existsSync, mkdirSync, renameSync } from 'node:fs';
|
|
8
8
|
import { cosmiconfig } from 'cosmiconfig';
|
|
9
|
-
import { ZodError } from 'zod';
|
|
10
|
-
import { readdir, stat, access } from 'node:fs/promises';
|
|
11
|
-
import { parallel } from 'radash';
|
|
12
|
-
import ignore from 'ignore';
|
|
13
|
-
import picomatch from 'picomatch';
|
|
14
|
-
import { execFile } from 'node:child_process';
|
|
15
|
-
import { promisify } from 'node:util';
|
|
9
|
+
import { ZodError, z } from 'zod';
|
|
16
10
|
|
|
17
11
|
/**
|
|
18
12
|
* @module api/handlers/configMerge
|
|
19
13
|
* Shared config merge utilities.
|
|
20
14
|
*/
|
|
21
|
-
/**
|
|
22
|
-
* Check whether a value is a plain object (not null, not an array).
|
|
23
|
-
*
|
|
24
|
-
* @param value - Value to check.
|
|
25
|
-
* @returns True for plain objects.
|
|
26
|
-
*/
|
|
27
|
-
function isPlainObject(value) {
|
|
28
|
-
return typeof value === 'object' && value !== null && !Array.isArray(value);
|
|
29
|
-
}
|
|
30
15
|
/**
|
|
31
16
|
* Deep-merge a config patch into a base config.
|
|
32
17
|
*
|
|
@@ -48,7 +33,7 @@ function deepMergeConfig(base, patch) {
|
|
|
48
33
|
? current
|
|
49
34
|
: undefined, value);
|
|
50
35
|
}
|
|
51
|
-
else if (isPlainObject(current) && isPlainObject(value)) {
|
|
36
|
+
else if (isPlainObject$1(current) && isPlainObject$1(value)) {
|
|
52
37
|
result[key] = deepMergeConfig(current, value);
|
|
53
38
|
}
|
|
54
39
|
else {
|
|
@@ -83,6 +68,401 @@ function mergeInferenceRules(existing, incoming) {
|
|
|
83
68
|
return merged;
|
|
84
69
|
}
|
|
85
70
|
|
|
71
|
+
/**
|
|
72
|
+
* @module rules/coerceType
|
|
73
|
+
* Coerces values to target JSON Schema types.
|
|
74
|
+
*/
|
|
75
|
+
/**
|
|
76
|
+
* Coerce a value to a target type based on JSON Schema type.
|
|
77
|
+
*
|
|
78
|
+
* @param value - The value to coerce (typically a string from template interpolation).
|
|
79
|
+
* @param type - The target JSON Schema type.
|
|
80
|
+
* @returns The coerced value, or undefined if coercion fails.
|
|
81
|
+
*/
|
|
82
|
+
function coerceType(value, type) {
|
|
83
|
+
// Null and undefined always coerce to undefined
|
|
84
|
+
if (value === null || value === undefined) {
|
|
85
|
+
return undefined;
|
|
86
|
+
}
|
|
87
|
+
// Empty strings are only valid for the 'string' type
|
|
88
|
+
if (value === '' && type !== 'string') {
|
|
89
|
+
return undefined;
|
|
90
|
+
}
|
|
91
|
+
switch (type) {
|
|
92
|
+
case 'string': {
|
|
93
|
+
if (typeof value === 'string')
|
|
94
|
+
return value;
|
|
95
|
+
if (typeof value === 'number' || typeof value === 'boolean') {
|
|
96
|
+
return String(value);
|
|
97
|
+
}
|
|
98
|
+
if (typeof value === 'object') {
|
|
99
|
+
// Stringify objects and arrays
|
|
100
|
+
return JSON.stringify(value);
|
|
101
|
+
}
|
|
102
|
+
// Unsupported types (symbol, function, etc.) - undefined
|
|
103
|
+
return undefined;
|
|
104
|
+
}
|
|
105
|
+
case 'integer': {
|
|
106
|
+
if (typeof value === 'string') {
|
|
107
|
+
// For strings, check that parsing yields an integer that matches the trimmed input
|
|
108
|
+
const trimmed = value.trim();
|
|
109
|
+
const num = parseInt(trimmed, 10);
|
|
110
|
+
if (Number.isInteger(num) && num.toString() === trimmed) {
|
|
111
|
+
return num;
|
|
112
|
+
}
|
|
113
|
+
return undefined;
|
|
114
|
+
}
|
|
115
|
+
if (typeof value === 'number') {
|
|
116
|
+
return Number.isInteger(value) ? value : undefined;
|
|
117
|
+
}
|
|
118
|
+
return undefined;
|
|
119
|
+
}
|
|
120
|
+
case 'number': {
|
|
121
|
+
const num = typeof value === 'string' ? parseFloat(value) : Number(value);
|
|
122
|
+
return Number.isFinite(num) ? num : undefined;
|
|
123
|
+
}
|
|
124
|
+
case 'boolean': {
|
|
125
|
+
if (typeof value === 'boolean')
|
|
126
|
+
return value;
|
|
127
|
+
if (typeof value === 'string') {
|
|
128
|
+
const lower = value.toLowerCase();
|
|
129
|
+
if (lower === 'true')
|
|
130
|
+
return true;
|
|
131
|
+
if (lower === 'false')
|
|
132
|
+
return false;
|
|
133
|
+
}
|
|
134
|
+
return undefined;
|
|
135
|
+
}
|
|
136
|
+
case 'array': {
|
|
137
|
+
if (Array.isArray(value))
|
|
138
|
+
return value;
|
|
139
|
+
if (typeof value === 'string') {
|
|
140
|
+
try {
|
|
141
|
+
const parsed = JSON.parse(value);
|
|
142
|
+
if (Array.isArray(parsed))
|
|
143
|
+
return parsed;
|
|
144
|
+
}
|
|
145
|
+
catch {
|
|
146
|
+
// Invalid JSON
|
|
147
|
+
}
|
|
148
|
+
}
|
|
149
|
+
return undefined;
|
|
150
|
+
}
|
|
151
|
+
case 'object': {
|
|
152
|
+
if (typeof value === 'string') {
|
|
153
|
+
try {
|
|
154
|
+
const parsed = JSON.parse(value);
|
|
155
|
+
if (typeof parsed === 'object' &&
|
|
156
|
+
parsed !== null &&
|
|
157
|
+
!Array.isArray(parsed)) {
|
|
158
|
+
return parsed;
|
|
159
|
+
}
|
|
160
|
+
}
|
|
161
|
+
catch {
|
|
162
|
+
// Invalid JSON
|
|
163
|
+
}
|
|
164
|
+
return undefined;
|
|
165
|
+
}
|
|
166
|
+
// Check for plain objects (not array) - null already filtered at top
|
|
167
|
+
if (typeof value === 'object' && !Array.isArray(value)) {
|
|
168
|
+
return value;
|
|
169
|
+
}
|
|
170
|
+
return undefined;
|
|
171
|
+
}
|
|
172
|
+
default:
|
|
173
|
+
return value;
|
|
174
|
+
}
|
|
175
|
+
}
|
|
176
|
+
|
|
177
|
+
/**
|
|
178
|
+
* @module rules/templates
|
|
179
|
+
* Resolves Handlebars template expressions in rule `set` objects against file attributes.
|
|
180
|
+
*/
|
|
181
|
+
/**
|
|
182
|
+
* Resolve a Handlebars template expression against file attributes.
|
|
183
|
+
*
|
|
184
|
+
* @param value - The value to resolve (strings are compiled as Handlebars templates).
|
|
185
|
+
* @param attributes - The file or source attributes for variable lookup.
|
|
186
|
+
* @param hbs - Handlebars instance with registered helpers.
|
|
187
|
+
* @returns The resolved value.
|
|
188
|
+
*/
|
|
189
|
+
function resolveTemplateVars(value, attributes, hbs) {
|
|
190
|
+
if (typeof value !== 'string')
|
|
191
|
+
return value;
|
|
192
|
+
return hbs.compile(value, { noEscape: true })(attributes);
|
|
193
|
+
}
|
|
194
|
+
|
|
195
|
+
/**
|
|
196
|
+
* @module rules/schemaMerge
|
|
197
|
+
* Merges schema references into a resolved property schema with set extraction and type coercion.
|
|
198
|
+
*/
|
|
199
|
+
/**
|
|
200
|
+
* Resolve a schema entry (inline object or file path) to a schema object.
|
|
201
|
+
*
|
|
202
|
+
* @param entry - Schema entry (inline object or file path).
|
|
203
|
+
* @param configDir - Config directory for resolving file paths.
|
|
204
|
+
* @returns The resolved schema object.
|
|
205
|
+
*/
|
|
206
|
+
function resolveSchemaEntry(entry, configDir) {
|
|
207
|
+
if (typeof entry === 'string') {
|
|
208
|
+
// File path - load and parse JSON
|
|
209
|
+
const resolvedPath = configDir ? resolve(configDir, entry) : entry;
|
|
210
|
+
const raw = readFileSync(resolvedPath, 'utf-8');
|
|
211
|
+
return JSON.parse(raw);
|
|
212
|
+
}
|
|
213
|
+
return entry;
|
|
214
|
+
}
|
|
215
|
+
/**
|
|
216
|
+
* Merge an array of schema references into a single resolved schema.
|
|
217
|
+
* Later entries override earlier ones at the property level.
|
|
218
|
+
*
|
|
219
|
+
* @param refs - Array of schema references (named strings or inline objects).
|
|
220
|
+
* @param options - Options for schema merging.
|
|
221
|
+
* @returns The merged resolved schema.
|
|
222
|
+
*/
|
|
223
|
+
function mergeSchemas(refs, options = {}) {
|
|
224
|
+
const { globalSchemas = {}, configDir } = options;
|
|
225
|
+
const merged = { properties: {} };
|
|
226
|
+
for (const ref of refs) {
|
|
227
|
+
let schema;
|
|
228
|
+
if (typeof ref === 'string') {
|
|
229
|
+
// Named reference - resolve from global schemas
|
|
230
|
+
const globalEntry = globalSchemas[ref];
|
|
231
|
+
if (!globalEntry) {
|
|
232
|
+
throw new Error(`Schema reference "${ref}" not found in global schemas`);
|
|
233
|
+
}
|
|
234
|
+
schema = resolveSchemaEntry(globalEntry, configDir);
|
|
235
|
+
}
|
|
236
|
+
else {
|
|
237
|
+
// Inline object
|
|
238
|
+
schema = ref;
|
|
239
|
+
}
|
|
240
|
+
// Merge properties at the property level
|
|
241
|
+
if (schema.properties) {
|
|
242
|
+
for (const [propName, propDef] of Object.entries(schema.properties)) {
|
|
243
|
+
const existing = merged.properties[propName];
|
|
244
|
+
if (existing) {
|
|
245
|
+
const incoming = propDef;
|
|
246
|
+
// For array-typed properties, concatenate set values instead of replacing
|
|
247
|
+
if (existing.type === 'array' &&
|
|
248
|
+
Array.isArray(existing.set) &&
|
|
249
|
+
incoming.set !== undefined) {
|
|
250
|
+
const existingSet = existing.set;
|
|
251
|
+
const incomingSet = Array.isArray(incoming.set)
|
|
252
|
+
? incoming.set
|
|
253
|
+
: [incoming.set];
|
|
254
|
+
merged.properties[propName] = {
|
|
255
|
+
...existing,
|
|
256
|
+
...propDef,
|
|
257
|
+
set: existingSet.concat(incomingSet),
|
|
258
|
+
};
|
|
259
|
+
}
|
|
260
|
+
else {
|
|
261
|
+
merged.properties[propName] = {
|
|
262
|
+
...existing,
|
|
263
|
+
...propDef,
|
|
264
|
+
};
|
|
265
|
+
}
|
|
266
|
+
}
|
|
267
|
+
else {
|
|
268
|
+
merged.properties[propName] = propDef;
|
|
269
|
+
}
|
|
270
|
+
}
|
|
271
|
+
}
|
|
272
|
+
}
|
|
273
|
+
return merged;
|
|
274
|
+
}
|
|
275
|
+
/**
|
|
276
|
+
* Resolve set templates against file attributes and coerce to declared types.
|
|
277
|
+
*
|
|
278
|
+
* @param schema - Resolved schema with type information.
|
|
279
|
+
* @param attributes - File attributes for template variable resolution.
|
|
280
|
+
* @returns Resolved and type-coerced metadata.
|
|
281
|
+
*/
|
|
282
|
+
function resolveAndCoerce(schema, attributes, hbs) {
|
|
283
|
+
const result = {};
|
|
284
|
+
for (const [propName, propDef] of Object.entries(schema.properties)) {
|
|
285
|
+
const setTemplate = propDef.set;
|
|
286
|
+
if (setTemplate === undefined)
|
|
287
|
+
continue;
|
|
288
|
+
// For array-typed properties with array set values, resolve each element
|
|
289
|
+
if (Array.isArray(setTemplate)) {
|
|
290
|
+
const resolved = setTemplate.map((item) => resolveTemplateVars(item, attributes, hbs));
|
|
291
|
+
result[propName] = resolved;
|
|
292
|
+
continue;
|
|
293
|
+
}
|
|
294
|
+
// Resolve template
|
|
295
|
+
const rawValue = resolveTemplateVars(setTemplate, attributes, hbs);
|
|
296
|
+
// Coerce to declared type - returns undefined on failure
|
|
297
|
+
const coerced = coerceType(rawValue, propDef.type);
|
|
298
|
+
// Only include if coercion succeeded
|
|
299
|
+
if (coerced !== undefined) {
|
|
300
|
+
result[propName] = coerced;
|
|
301
|
+
}
|
|
302
|
+
}
|
|
303
|
+
return result;
|
|
304
|
+
}
|
|
305
|
+
/**
|
|
306
|
+
* Validate that all properties in a schema have a declared type.
|
|
307
|
+
* Throws if any property lacks a type.
|
|
308
|
+
*
|
|
309
|
+
* @param schema - Resolved schema to validate.
|
|
310
|
+
* @param ruleName - Name of the rule (for error messages).
|
|
311
|
+
*/
|
|
312
|
+
function validateSchemaCompleteness(schema, ruleName) {
|
|
313
|
+
for (const [propName, propDef] of Object.entries(schema.properties)) {
|
|
314
|
+
if (!propDef.type) {
|
|
315
|
+
throw new Error(`Property "${propName}" in rule "${ruleName}" has no declared type. Every property must have a type.`);
|
|
316
|
+
}
|
|
317
|
+
}
|
|
318
|
+
}
|
|
319
|
+
/** Types that can produce trackable facet values. */
|
|
320
|
+
const FACETABLE_TYPES = new Set([
|
|
321
|
+
'string',
|
|
322
|
+
'number',
|
|
323
|
+
'boolean',
|
|
324
|
+
'integer',
|
|
325
|
+
'array',
|
|
326
|
+
]);
|
|
327
|
+
/**
|
|
328
|
+
* Validate that uiHint and enum are only applied to facetable property types.
|
|
329
|
+
* Throws if a non-facetable type (e.g. object) declares uiHint or enum.
|
|
330
|
+
*
|
|
331
|
+
* @param schema - Resolved schema to validate.
|
|
332
|
+
* @param ruleName - Name of the rule (for error messages).
|
|
333
|
+
*/
|
|
334
|
+
function validateFacetTypes(schema, ruleName) {
|
|
335
|
+
for (const [propName, propDef] of Object.entries(schema.properties)) {
|
|
336
|
+
if (!propDef.type)
|
|
337
|
+
continue;
|
|
338
|
+
if (!FACETABLE_TYPES.has(propDef.type) &&
|
|
339
|
+
(propDef.uiHint !== undefined || propDef.enum !== undefined)) {
|
|
340
|
+
throw new Error(`Property "${propName}" in rule "${ruleName}" has type "${propDef.type}" with uiHint/enum. Facet hints are only valid on string, number, boolean, integer, or array types.`);
|
|
341
|
+
}
|
|
342
|
+
}
|
|
343
|
+
}
|
|
344
|
+
|
|
345
|
+
/**
|
|
346
|
+
* @module rules/reservedFieldWarnings
|
|
347
|
+
* Detects inference-rule schema properties named like reserved system payload fields and reports them as non-fatal config warnings (emitted by loadConfig and `/config/validate`). Reads file-based schemas via mergeSchemas.
|
|
348
|
+
*/
|
|
349
|
+
/**
|
|
350
|
+
* Find rule schema properties that use a reserved system field name.
|
|
351
|
+
*
|
|
352
|
+
* Rules whose schemas cannot be resolved are skipped: schema resolution
|
|
353
|
+
* failures are reported as errors by validation, not as warnings here.
|
|
354
|
+
*
|
|
355
|
+
* @param config - Config (or candidate config) with rules and global schemas.
|
|
356
|
+
* @param configDir - Directory used to resolve file-based schema paths.
|
|
357
|
+
* @returns One warning per offending rule property.
|
|
358
|
+
*/
|
|
359
|
+
function findReservedFieldWarnings(config, configDir) {
|
|
360
|
+
const warnings = [];
|
|
361
|
+
for (const rule of config.inferenceRules ?? []) {
|
|
362
|
+
if (typeof rule === 'string' || !rule.schema?.length)
|
|
363
|
+
continue;
|
|
364
|
+
let properties;
|
|
365
|
+
try {
|
|
366
|
+
properties = mergeSchemas(rule.schema, {
|
|
367
|
+
globalSchemas: config.schemas,
|
|
368
|
+
configDir,
|
|
369
|
+
}).properties;
|
|
370
|
+
}
|
|
371
|
+
catch {
|
|
372
|
+
continue;
|
|
373
|
+
}
|
|
374
|
+
for (const name of findReservedPropertyNames(properties)) {
|
|
375
|
+
warnings.push({
|
|
376
|
+
path: `inferenceRules[${rule.name}].schema.${name}`,
|
|
377
|
+
message: `Property "${name}" is a reserved system payload field; the system value overwrites rule metadata on upsert. Rename the property.`,
|
|
378
|
+
});
|
|
379
|
+
}
|
|
380
|
+
}
|
|
381
|
+
return warnings;
|
|
382
|
+
}
|
|
383
|
+
|
|
384
|
+
/**
|
|
385
|
+
* @module config/assertResolvedSecrets
|
|
386
|
+
* Fail-fast check that secret-bearing config values (api.apiKey, sources.*.url and headers) contain no unresolved `${VAR}` after env substitution, and that api.apiKey is not empty. Pure; throws.
|
|
387
|
+
*/
|
|
388
|
+
const UNRESOLVED_PATTERN = /\$\{[^}]*\}/;
|
|
389
|
+
/**
|
|
390
|
+
* Collect config paths whose secret-bearing value still contains an
|
|
391
|
+
* unresolved `${…}` reference. Values themselves are never returned.
|
|
392
|
+
*
|
|
393
|
+
* @param config - Config after environment-variable substitution.
|
|
394
|
+
* @returns Dotted paths of offending values.
|
|
395
|
+
*/
|
|
396
|
+
function findUnresolvedSecrets(config) {
|
|
397
|
+
const paths = [];
|
|
398
|
+
const check = (path, value) => {
|
|
399
|
+
if (value !== undefined && UNRESOLVED_PATTERN.test(value))
|
|
400
|
+
paths.push(path);
|
|
401
|
+
};
|
|
402
|
+
check('api.apiKey', config.api?.apiKey);
|
|
403
|
+
for (const [name, source] of Object.entries(config.sources ?? {})) {
|
|
404
|
+
check(`sources.${name}.url`, source.url);
|
|
405
|
+
for (const [header, value] of Object.entries(source.headers ?? {})) {
|
|
406
|
+
check(`sources.${name}.headers.${header}`, value);
|
|
407
|
+
}
|
|
408
|
+
}
|
|
409
|
+
return paths;
|
|
410
|
+
}
|
|
411
|
+
/**
|
|
412
|
+
* Throw when any secret-bearing value still contains an unresolved `${…}`,
|
|
413
|
+
* or when `api.apiKey` is empty after substitution (e.g. its variable is set
|
|
414
|
+
* to an empty string). Errors name config paths only, never values.
|
|
415
|
+
*
|
|
416
|
+
* @param config - Config after environment-variable substitution.
|
|
417
|
+
* @throws Error listing the offending config paths.
|
|
418
|
+
*/
|
|
419
|
+
function assertResolvedSecrets(config) {
|
|
420
|
+
const paths = findUnresolvedSecrets(config);
|
|
421
|
+
if (paths.length > 0) {
|
|
422
|
+
throw new Error(`Unresolved environment variable reference in: ${paths.join(', ')}. Set the referenced environment variables before starting jeeves-watcher.`);
|
|
423
|
+
}
|
|
424
|
+
if (config.api?.apiKey === '') {
|
|
425
|
+
throw new Error('api.apiKey is empty after environment variable substitution. Set a non-empty key, or remove api.apiKey to disable authentication.');
|
|
426
|
+
}
|
|
427
|
+
}
|
|
428
|
+
|
|
429
|
+
/**
|
|
430
|
+
* @module config/sourceKeyWarnings
|
|
431
|
+
* Finds unknown keys under `sources.<name>` in a raw (unparsed) config, e.g. a misspelt or unsupported `batchUrl`, which the schema would silently drop. Pure.
|
|
432
|
+
*/
|
|
433
|
+
/** Keys a source entry may carry. */
|
|
434
|
+
const KNOWN_SOURCE_KEYS = new Set(Object.keys(sourceConfigSchema.shape));
|
|
435
|
+
/**
|
|
436
|
+
* Find keys under `sources.<name>` that the source schema does not define.
|
|
437
|
+
* Validation strips them, so without this warning a setting such as
|
|
438
|
+
* `batchUrl` would be ignored without notice.
|
|
439
|
+
*
|
|
440
|
+
* @param rawConfig - The config as written (before schema parsing).
|
|
441
|
+
* @returns One warning per unknown key, in config order.
|
|
442
|
+
*/
|
|
443
|
+
function findUnknownSourceKeys(rawConfig) {
|
|
444
|
+
if (!isPlainObject$1(rawConfig) || !isPlainObject$1(rawConfig.sources))
|
|
445
|
+
return [];
|
|
446
|
+
const warnings = [];
|
|
447
|
+
for (const [name, source] of Object.entries(rawConfig.sources)) {
|
|
448
|
+
if (!isPlainObject$1(source))
|
|
449
|
+
continue;
|
|
450
|
+
for (const key of Object.keys(source)) {
|
|
451
|
+
if (KNOWN_SOURCE_KEYS.has(key))
|
|
452
|
+
continue;
|
|
453
|
+
warnings.push({
|
|
454
|
+
path: `sources.${name}.${key}`,
|
|
455
|
+
message: `Unknown source setting "${key}" is ignored (supported: ${[...KNOWN_SOURCE_KEYS].join(', ')}).`,
|
|
456
|
+
});
|
|
457
|
+
}
|
|
458
|
+
}
|
|
459
|
+
return warnings;
|
|
460
|
+
}
|
|
461
|
+
|
|
462
|
+
/**
|
|
463
|
+
* @module config/loadConfig
|
|
464
|
+
* Loads the watcher config via cosmiconfig: schema validation, rule/map file resolution, defaults, env substitution and secret checks; reports non-fatal warnings (unknown source keys, reserved rule field names) through onWarning. Reads files.
|
|
465
|
+
*/
|
|
86
466
|
const MODULE_NAME = 'jeeves-watcher';
|
|
87
467
|
/**
|
|
88
468
|
* Merge sensible defaults into a loaded configuration.
|
|
@@ -105,10 +485,11 @@ function applyDefaults(raw) {
|
|
|
105
485
|
* Load the jeeves-watcher configuration.
|
|
106
486
|
*
|
|
107
487
|
* @param configPath - Optional explicit path to a config file.
|
|
488
|
+
* @param options - Optional warning sink.
|
|
108
489
|
* @returns The loaded configuration.
|
|
109
490
|
* @throws If no configuration is found or validation fails.
|
|
110
491
|
*/
|
|
111
|
-
async function loadConfig(configPath) {
|
|
492
|
+
async function loadConfig(configPath, options = {}) {
|
|
112
493
|
const explorer = cosmiconfig(MODULE_NAME);
|
|
113
494
|
const result = configPath
|
|
114
495
|
? await explorer.load(configPath)
|
|
@@ -145,7 +526,15 @@ async function loadConfig(configPath) {
|
|
|
145
526
|
}
|
|
146
527
|
}
|
|
147
528
|
const withDefaults = applyDefaults(validated);
|
|
148
|
-
|
|
529
|
+
const substituted = substituteEnvVars(withDefaults);
|
|
530
|
+
assertResolvedSecrets(substituted);
|
|
531
|
+
const warnings = [
|
|
532
|
+
...findUnknownSourceKeys(result.config),
|
|
533
|
+
...findReservedFieldWarnings(substituted, configDir),
|
|
534
|
+
];
|
|
535
|
+
for (const warning of warnings)
|
|
536
|
+
options.onWarning?.(warning);
|
|
537
|
+
return substituted;
|
|
149
538
|
}
|
|
150
539
|
catch (error) {
|
|
151
540
|
if (error instanceof ZodError) {
|
|
@@ -209,8 +598,98 @@ function migrateConfigPath(configDir) {
|
|
|
209
598
|
|
|
210
599
|
/**
|
|
211
600
|
* @module app/shutdown
|
|
212
|
-
*
|
|
601
|
+
* Shutdown orchestration: closes the API server first (no new notifies), stops producers, drains and stops queues with timeouts, and closes stores only when no abandoned run is still in flight. Installs SIGINT/SIGTERM handlers.
|
|
602
|
+
*/
|
|
603
|
+
/**
|
|
604
|
+
* Wait for a promise, giving up after `timeoutMs`.
|
|
605
|
+
*
|
|
606
|
+
* @param promise - The work to wait for (its rejection counts as settled).
|
|
607
|
+
* @param timeoutMs - Maximum wait in milliseconds.
|
|
608
|
+
* @returns `true` when settled in time, `false` on timeout.
|
|
609
|
+
*/
|
|
610
|
+
async function settlesWithin(promise, timeoutMs) {
|
|
611
|
+
let timer;
|
|
612
|
+
const settled = await Promise.race([
|
|
613
|
+
promise.then(() => true, () => true),
|
|
614
|
+
new Promise((resolve) => {
|
|
615
|
+
timer = setTimeout(() => {
|
|
616
|
+
resolve(false);
|
|
617
|
+
}, timeoutMs);
|
|
618
|
+
}),
|
|
619
|
+
]);
|
|
620
|
+
clearTimeout(timer);
|
|
621
|
+
return settled;
|
|
622
|
+
}
|
|
623
|
+
/**
|
|
624
|
+
* Wait for a step, warning when it times out.
|
|
625
|
+
*
|
|
626
|
+
* @returns `true` when the step settled in time.
|
|
627
|
+
*/
|
|
628
|
+
async function step(label, work, timeoutMs, logger) {
|
|
629
|
+
const done = await settlesWithin(work, timeoutMs);
|
|
630
|
+
if (!done) {
|
|
631
|
+
logger?.warn({ timeoutMs, step: label }, `${label} timeout hit, forcing shutdown`);
|
|
632
|
+
}
|
|
633
|
+
return done;
|
|
634
|
+
}
|
|
635
|
+
/**
|
|
636
|
+
* Drain a queue, then stop it (whatever the drain did); each step bounded.
|
|
637
|
+
*
|
|
638
|
+
* @returns `true` when the stop finished, i.e. no run is still in flight.
|
|
639
|
+
*/
|
|
640
|
+
async function drainThenStop(label, queue, timeoutMs, logger) {
|
|
641
|
+
await step(`${label} drain`, queue.drain(), timeoutMs, logger);
|
|
642
|
+
return step(`${label} stop`, queue.stop(), timeoutMs, logger);
|
|
643
|
+
}
|
|
644
|
+
/**
|
|
645
|
+
* Drain and stop the file queue, then the source notify queue. Stopping
|
|
646
|
+
* drops work the drain did not reach, refuses new work and waits for
|
|
647
|
+
* in-flight runs; each step is bounded by `timeoutMs`.
|
|
648
|
+
*
|
|
649
|
+
* @param queues - The queues (absent ones are skipped).
|
|
650
|
+
* @param timeoutMs - Per-step timeout in milliseconds.
|
|
651
|
+
* @param logger - Logger for timeout warnings.
|
|
652
|
+
* @returns `true` when every stop finished, i.e. no run is still in flight.
|
|
653
|
+
*/
|
|
654
|
+
async function drainQueues(queues, timeoutMs, logger) {
|
|
655
|
+
let idle = true;
|
|
656
|
+
if (queues.queue) {
|
|
657
|
+
idle = await drainThenStop('Queue', queues.queue, timeoutMs, logger);
|
|
658
|
+
}
|
|
659
|
+
if (queues.sourceQueue) {
|
|
660
|
+
const { sourceQueue } = queues;
|
|
661
|
+
const stopped = await drainThenStop('Source queue', sourceQueue, timeoutMs, logger);
|
|
662
|
+
idle = stopped && idle;
|
|
663
|
+
}
|
|
664
|
+
return idle;
|
|
665
|
+
}
|
|
666
|
+
/**
|
|
667
|
+
* Stop every component in a safe order: close the API server (bounded), stop
|
|
668
|
+
* producers, drain and stop the queues (bounded), flush the values index, and
|
|
669
|
+
* close stores. When a queue step timed out, runs may still be in flight, so
|
|
670
|
+
* stores are left open (the process exits) rather than closed under them.
|
|
671
|
+
*
|
|
672
|
+
* @param components - The components.
|
|
673
|
+
* @param timeoutMs - Per-step timeout in milliseconds.
|
|
674
|
+
* @param logger - Logger for warnings.
|
|
213
675
|
*/
|
|
676
|
+
async function shutdownComponents(components, timeoutMs, logger) {
|
|
677
|
+
const { server, producers = [], valuesManager, stores = [] } = components;
|
|
678
|
+
if (server)
|
|
679
|
+
await step('API server close', server.close(), timeoutMs, logger);
|
|
680
|
+
for (const producer of producers)
|
|
681
|
+
await producer?.stop();
|
|
682
|
+
const idle = await drainQueues(components, timeoutMs, logger);
|
|
683
|
+
valuesManager?.stopAutoFlush();
|
|
684
|
+
valuesManager?.flush();
|
|
685
|
+
if (idle) {
|
|
686
|
+
for (const store of stores)
|
|
687
|
+
store?.close();
|
|
688
|
+
}
|
|
689
|
+
else {
|
|
690
|
+
logger?.warn('Runs still in flight after shutdown timeout; stores left open');
|
|
691
|
+
}
|
|
692
|
+
}
|
|
214
693
|
/**
|
|
215
694
|
* Install process signal handlers.
|
|
216
695
|
*
|
|
@@ -262,10 +741,16 @@ async function startFromConfig(configPath, descriptor) {
|
|
|
262
741
|
resolvedPath = undefined;
|
|
263
742
|
}
|
|
264
743
|
}
|
|
265
|
-
|
|
744
|
+
// The logger is built from the config, so load-time warnings go to the
|
|
745
|
+
// console like the migration warnings above.
|
|
746
|
+
const config = await loadConfig(resolvedPath, {
|
|
747
|
+
onWarning: (warning) => {
|
|
748
|
+
console.warn(`[jeeves-watcher] ${warning.path}: ${warning.message}`);
|
|
749
|
+
},
|
|
750
|
+
});
|
|
266
751
|
// Dynamic import breaks the circular dependency between this module
|
|
267
752
|
// and app/index.ts (which defines JeevesWatcher and re-exports startFromConfig).
|
|
268
|
-
const { JeevesWatcher } = await import('./index-
|
|
753
|
+
const { JeevesWatcher } = await import('./index-C9-xTM7k.js');
|
|
269
754
|
const app = new JeevesWatcher(config, resolvedPath, descriptor);
|
|
270
755
|
installShutdownHandlers(() => app.stop());
|
|
271
756
|
await app.start();
|
|
@@ -273,1151 +758,345 @@ async function startFromConfig(configPath, descriptor) {
|
|
|
273
758
|
}
|
|
274
759
|
|
|
275
760
|
/**
|
|
276
|
-
* @module
|
|
277
|
-
*
|
|
761
|
+
* @module cli/jeeves-watcher/apiClient
|
|
762
|
+
* Shared HTTP client for every CLI command that calls the watcher API: JSON GET/POST via `fetch` with the core bearer header and core error mapping (errors carry the server `code`; 401 adds key guidance). Never prints the key.
|
|
763
|
+
*
|
|
764
|
+
* @remarks
|
|
765
|
+
* Key resolution ({@link resolveApiKey}): the `--api-key` flag, then the
|
|
766
|
+
* `JEEVES_WATCHER_API_KEY` environment variable (core `API_KEY_ENV_VAR`). The
|
|
767
|
+
* API commands do not load the watcher config file, so `api.apiKey` from
|
|
768
|
+
* config is not consulted.
|
|
278
769
|
*/
|
|
770
|
+
/** Where the CLI reads its API key from (named in 401 guidance). */
|
|
771
|
+
const CLI_KEY_SOURCES = `--api-key or the ${API_KEY_ENV_VAR} env var`;
|
|
772
|
+
/** Options for {@link createApiClient}; the Zod schema is the source of truth. */
|
|
773
|
+
const apiClientOptionsSchema = z.object({
|
|
774
|
+
/** API base URL, e.g. `http://127.0.0.1:1936`. */
|
|
775
|
+
baseUrl: z.string().min(1),
|
|
776
|
+
/** Bearer key; omitted = no `Authorization` header. */
|
|
777
|
+
apiKey: z.string().min(1).optional(),
|
|
778
|
+
});
|
|
779
|
+
/** Treat missing and blank values as unset. */
|
|
780
|
+
function nonBlank(value) {
|
|
781
|
+
return value !== undefined && value.trim() !== '' ? value : undefined;
|
|
782
|
+
}
|
|
279
783
|
/**
|
|
280
|
-
*
|
|
281
|
-
*
|
|
784
|
+
* Resolve the API key for a CLI invocation. Pure.
|
|
785
|
+
*
|
|
786
|
+
* @param flag - Value of the `--api-key` option, if given.
|
|
787
|
+
* @param env - Environment to read `API_KEY_ENV_VAR` from.
|
|
788
|
+
* @returns The first non-blank of `flag` and the env var, or `undefined`.
|
|
282
789
|
*/
|
|
283
|
-
function
|
|
284
|
-
|
|
285
|
-
const root = resolve('/');
|
|
286
|
-
while (dir !== root) {
|
|
287
|
-
if (existsSync(join(dir, '.git')) &&
|
|
288
|
-
statSync(join(dir, '.git')).isDirectory()) {
|
|
289
|
-
return dir;
|
|
290
|
-
}
|
|
291
|
-
const parent = dirname(dir);
|
|
292
|
-
if (parent === dir)
|
|
293
|
-
break;
|
|
294
|
-
dir = parent;
|
|
295
|
-
}
|
|
296
|
-
return undefined;
|
|
790
|
+
function resolveApiKey(flag, env = process.env) {
|
|
791
|
+
return nonBlank(flag) ?? nonBlank(env[API_KEY_ENV_VAR]);
|
|
297
792
|
}
|
|
298
793
|
/**
|
|
299
|
-
*
|
|
300
|
-
*
|
|
794
|
+
* Error text for a `401` CLI response (core `unauthorizedMessage` naming
|
|
795
|
+
* {@link CLI_KEY_SOURCES}). Never contains the key. Pure.
|
|
796
|
+
*
|
|
797
|
+
* @param keySent - Whether the request carried a bearer key.
|
|
798
|
+
* @returns Guidance on supplying (or checking) the key.
|
|
301
799
|
*/
|
|
302
|
-
function
|
|
303
|
-
|
|
304
|
-
try {
|
|
305
|
-
return statSync(absPath).isDirectory() ? absPath : dirname(absPath);
|
|
306
|
-
}
|
|
307
|
-
catch {
|
|
308
|
-
// ignore
|
|
309
|
-
}
|
|
310
|
-
// If this is a glob, fall back to the non-glob prefix.
|
|
311
|
-
const globMatch = /[*?[{]/.exec(watchPath);
|
|
312
|
-
if (!globMatch)
|
|
313
|
-
return undefined;
|
|
314
|
-
const prefix = watchPath.slice(0, globMatch.index);
|
|
315
|
-
const trimmed = prefix.trim();
|
|
316
|
-
const baseDir = trimmed.length === 0
|
|
317
|
-
? '.'
|
|
318
|
-
: trimmed.endsWith('/') || trimmed.endsWith('\\')
|
|
319
|
-
? trimmed
|
|
320
|
-
: dirname(trimmed);
|
|
321
|
-
const resolved = resolve(baseDir);
|
|
322
|
-
if (!existsSync(resolved))
|
|
323
|
-
return undefined;
|
|
324
|
-
return resolved;
|
|
800
|
+
function cliUnauthorizedMessage(keySent) {
|
|
801
|
+
return unauthorizedMessage(keySent, CLI_KEY_SOURCES);
|
|
325
802
|
}
|
|
326
803
|
/**
|
|
327
|
-
*
|
|
328
|
-
*
|
|
804
|
+
* Send one request; map 401 and other non-OK statuses to errors.
|
|
805
|
+
*
|
|
806
|
+
* @param url - Full request URL.
|
|
807
|
+
* @param init - Request init.
|
|
808
|
+
* @param keySent - Whether the request carries a bearer key.
|
|
809
|
+
* @returns The parsed JSON body.
|
|
810
|
+
* @throws On `401` with {@link cliUnauthorizedMessage}; on any other non-OK
|
|
811
|
+
* status with
|
|
812
|
+
* `HTTP <status> <code>: <message>`.
|
|
329
813
|
*/
|
|
330
|
-
function
|
|
331
|
-
const
|
|
332
|
-
|
|
333
|
-
|
|
334
|
-
results.push(gitignorePath);
|
|
814
|
+
async function send(url, init, keySent) {
|
|
815
|
+
const res = await fetch(url, init);
|
|
816
|
+
if (res.status === 401) {
|
|
817
|
+
throw new Error(cliUnauthorizedMessage(keySent));
|
|
335
818
|
}
|
|
336
|
-
|
|
337
|
-
|
|
338
|
-
|
|
339
|
-
}
|
|
340
|
-
catch {
|
|
341
|
-
return results;
|
|
342
|
-
}
|
|
343
|
-
for (const entry of entries) {
|
|
344
|
-
if (entry === '.git' || entry === 'node_modules')
|
|
345
|
-
continue;
|
|
346
|
-
const fullPath = join(dir, entry);
|
|
347
|
-
try {
|
|
348
|
-
if (statSync(fullPath).isDirectory()) {
|
|
349
|
-
results.push(...findGitignoreFiles(fullPath));
|
|
350
|
-
}
|
|
351
|
-
}
|
|
352
|
-
catch {
|
|
353
|
-
// Skip inaccessible entries
|
|
354
|
-
}
|
|
819
|
+
if (!res.ok) {
|
|
820
|
+
const info = httpError(res.status, res.statusText, await res.text());
|
|
821
|
+
throw new Error(formatHttpError(info));
|
|
355
822
|
}
|
|
356
|
-
return
|
|
823
|
+
return res.json();
|
|
357
824
|
}
|
|
358
825
|
/**
|
|
359
|
-
*
|
|
826
|
+
* Create a JSON client for the watcher API.
|
|
827
|
+
*
|
|
828
|
+
* @param options - Base URL and optional bearer key.
|
|
829
|
+
* @returns A client whose requests carry `Authorization: Bearer <apiKey>`
|
|
830
|
+
* when a key is set, and no `Authorization` header otherwise.
|
|
360
831
|
*/
|
|
361
|
-
function
|
|
362
|
-
const
|
|
363
|
-
|
|
832
|
+
function createApiClient(options) {
|
|
833
|
+
const { baseUrl, apiKey } = apiClientOptionsSchema.parse(options);
|
|
834
|
+
const keySent = apiKey !== undefined;
|
|
835
|
+
const auth = bearerHeaders(apiKey);
|
|
836
|
+
return {
|
|
837
|
+
get: (path) => send(`${baseUrl}${path}`, keySent ? { headers: auth } : undefined, keySent),
|
|
838
|
+
post: (path, body) => send(`${baseUrl}${path}`, {
|
|
839
|
+
method: 'POST',
|
|
840
|
+
headers: { 'Content-Type': 'application/json', ...auth },
|
|
841
|
+
body: JSON.stringify(body),
|
|
842
|
+
}, keySent),
|
|
843
|
+
};
|
|
364
844
|
}
|
|
845
|
+
|
|
365
846
|
/**
|
|
366
|
-
*
|
|
847
|
+
* @module cli/jeeves-watcher/apiOptions
|
|
848
|
+
* Commander plumbing shared by the API-calling CLI commands: --port/--host/--api-key options, API client construction, error handling and JSON output.
|
|
367
849
|
*/
|
|
368
|
-
|
|
369
|
-
|
|
370
|
-
}
|
|
850
|
+
const DEFAULT_PORT = String(API_DEFAULTS.port);
|
|
851
|
+
const DEFAULT_HOST = API_DEFAULTS.host;
|
|
371
852
|
/**
|
|
372
|
-
*
|
|
373
|
-
*
|
|
374
|
-
* Eliminates the repeated inline pattern:
|
|
375
|
-
* ```ts
|
|
376
|
-
* gitignoreFilter ? (filePath: string) => gitignoreFilter!.isIgnored(filePath) : undefined
|
|
377
|
-
* ```
|
|
378
|
-
*
|
|
379
|
-
* @param filter - Optional gitignore filter instance.
|
|
380
|
-
* @returns A callback that checks gitignore status, or `undefined` when no filter is available.
|
|
853
|
+
* Add the standard `--port`, `--host` and `--api-key` options to a command.
|
|
854
|
+
* The key has no default, so it never appears in `--help` output.
|
|
381
855
|
*/
|
|
382
|
-
function
|
|
383
|
-
return
|
|
856
|
+
function withApiOptions(cmd) {
|
|
857
|
+
return cmd
|
|
858
|
+
.option('-p, --port <port>', 'API port', DEFAULT_PORT)
|
|
859
|
+
.option('-H, --host <host>', 'API host', DEFAULT_HOST)
|
|
860
|
+
.option('--api-key <key>', `API bearer key (defaults to the ${API_KEY_ENV_VAR} environment variable)`);
|
|
384
861
|
}
|
|
385
862
|
/**
|
|
386
|
-
*
|
|
387
|
-
*
|
|
863
|
+
* Build the API client for a command invocation.
|
|
864
|
+
*
|
|
865
|
+
* @param opts - Parsed command options.
|
|
866
|
+
* @returns A client for `http://<host>:<port>` carrying the resolved key.
|
|
388
867
|
*/
|
|
389
|
-
|
|
390
|
-
|
|
391
|
-
|
|
392
|
-
|
|
393
|
-
|
|
394
|
-
|
|
395
|
-
|
|
396
|
-
|
|
397
|
-
|
|
398
|
-
|
|
399
|
-
|
|
400
|
-
* Scan paths for git repos and their `.gitignore` files.
|
|
401
|
-
*/
|
|
402
|
-
scan(watchPaths) {
|
|
403
|
-
this.repos.clear();
|
|
404
|
-
const scannedDirs = new Set();
|
|
405
|
-
for (const watchPath of watchPaths) {
|
|
406
|
-
const scanDir = watchPathToScanDir(watchPath);
|
|
407
|
-
if (!scanDir)
|
|
408
|
-
continue;
|
|
409
|
-
if (scannedDirs.has(scanDir))
|
|
410
|
-
continue;
|
|
411
|
-
scannedDirs.add(scanDir);
|
|
412
|
-
const repoRoot = findRepoRoot(scanDir);
|
|
413
|
-
if (!repoRoot) {
|
|
414
|
-
// No git repo found. Check for a standalone .gitignore at the scan dir.
|
|
415
|
-
// This handles directories like OpenClaw workspaces that have .gitignore
|
|
416
|
-
// but no .git directory.
|
|
417
|
-
const standaloneGitignore = join(scanDir, '.gitignore');
|
|
418
|
-
if (existsSync(standaloneGitignore) && !this.repos.has(scanDir)) {
|
|
419
|
-
const entry = {
|
|
420
|
-
dir: scanDir,
|
|
421
|
-
ig: parseGitignore(standaloneGitignore),
|
|
422
|
-
};
|
|
423
|
-
this.repos.set(scanDir, { root: scanDir, entries: [entry] });
|
|
424
|
-
}
|
|
425
|
-
continue;
|
|
426
|
-
}
|
|
427
|
-
if (this.repos.has(repoRoot))
|
|
428
|
-
continue;
|
|
429
|
-
const gitignoreFiles = findGitignoreFiles(repoRoot);
|
|
430
|
-
const entries = gitignoreFiles.map((gf) => ({
|
|
431
|
-
dir: dirname(gf),
|
|
432
|
-
ig: parseGitignore(gf),
|
|
433
|
-
}));
|
|
434
|
-
// Sort deepest-first so nested `.gitignore` files are checked first
|
|
435
|
-
entries.sort((a, b) => b.dir.length - a.dir.length);
|
|
436
|
-
this.repos.set(repoRoot, { root: repoRoot, entries });
|
|
437
|
-
}
|
|
438
|
-
}
|
|
439
|
-
/**
|
|
440
|
-
* Check whether a file path is ignored by any applicable `.gitignore`.
|
|
441
|
-
*
|
|
442
|
-
* @param filePath - Absolute file path to check.
|
|
443
|
-
* @returns `true` if the file should be ignored.
|
|
444
|
-
*/
|
|
445
|
-
isIgnored(filePath) {
|
|
446
|
-
const absPath = resolve(filePath);
|
|
447
|
-
for (const [, repo] of this.repos) {
|
|
448
|
-
// Check if file is within this repo
|
|
449
|
-
const relToRepo = relative(repo.root, absPath);
|
|
450
|
-
// On Windows, path.relative() across drives (e.g. D:\ → J:\) produces
|
|
451
|
-
// an absolute path with a drive letter instead of a relative one. The
|
|
452
|
-
// `ignore` library rejects these with a RangeError. Skip repos on
|
|
453
|
-
// different drives to avoid cross-drive gitignore mismatches.
|
|
454
|
-
if (relToRepo.startsWith('..') ||
|
|
455
|
-
relToRepo.startsWith(resolve('/')) ||
|
|
456
|
-
/^[a-zA-Z]:/.test(relToRepo)) {
|
|
457
|
-
continue;
|
|
458
|
-
}
|
|
459
|
-
// Check each `.gitignore` entry (deepest-first)
|
|
460
|
-
for (const entry of repo.entries) {
|
|
461
|
-
const relToEntry = relative(entry.dir, absPath);
|
|
462
|
-
if (relToEntry.startsWith('..') || /^[a-zA-Z]:/.test(relToEntry))
|
|
463
|
-
continue;
|
|
464
|
-
const normalized = toForwardSlash(relToEntry);
|
|
465
|
-
if (entry.ig.ignores(normalized)) {
|
|
466
|
-
return true;
|
|
467
|
-
}
|
|
468
|
-
}
|
|
469
|
-
}
|
|
470
|
-
return false;
|
|
471
|
-
}
|
|
472
|
-
/**
|
|
473
|
-
* Invalidate and re-parse a specific `.gitignore` file.
|
|
474
|
-
* Call when a `.gitignore` file is added, changed, or removed.
|
|
475
|
-
*
|
|
476
|
-
* @param gitignorePath - Absolute path to the `.gitignore` file that changed.
|
|
477
|
-
*/
|
|
478
|
-
invalidate(gitignorePath) {
|
|
479
|
-
const absPath = resolve(gitignorePath);
|
|
480
|
-
const gitignoreDir = dirname(absPath);
|
|
481
|
-
for (const [, repo] of this.repos) {
|
|
482
|
-
const relToRepo = relative(repo.root, gitignoreDir);
|
|
483
|
-
if (relToRepo.startsWith('..'))
|
|
484
|
-
continue;
|
|
485
|
-
// Remove old entry for this directory
|
|
486
|
-
repo.entries = repo.entries.filter((e) => e.dir !== gitignoreDir);
|
|
487
|
-
// Re-parse if file still exists
|
|
488
|
-
if (existsSync(absPath)) {
|
|
489
|
-
repo.entries.push({ dir: gitignoreDir, ig: parseGitignore(absPath) });
|
|
490
|
-
// Re-sort deepest-first
|
|
491
|
-
repo.entries.sort((a, b) => b.dir.length - a.dir.length);
|
|
492
|
-
}
|
|
493
|
-
return;
|
|
868
|
+
function apiClient(opts) {
|
|
869
|
+
return createApiClient({
|
|
870
|
+
baseUrl: `http://${opts.host}:${opts.port}`,
|
|
871
|
+
apiKey: resolveApiKey(opts.apiKey),
|
|
872
|
+
});
|
|
873
|
+
}
|
|
874
|
+
/** Wrap an async action with standard error handling (stderr + exit code 1). */
|
|
875
|
+
function handleErrors(fn) {
|
|
876
|
+
return async () => {
|
|
877
|
+
try {
|
|
878
|
+
await fn();
|
|
494
879
|
}
|
|
495
|
-
|
|
496
|
-
|
|
497
|
-
|
|
498
|
-
const entries = [
|
|
499
|
-
{ dir: gitignoreDir, ig: parseGitignore(absPath) },
|
|
500
|
-
];
|
|
501
|
-
if (this.repos.has(repoRoot)) {
|
|
502
|
-
const repo = this.repos.get(repoRoot);
|
|
503
|
-
repo.entries.push(entries[0]);
|
|
504
|
-
repo.entries.sort((a, b) => b.dir.length - a.dir.length);
|
|
505
|
-
}
|
|
506
|
-
else {
|
|
507
|
-
this.repos.set(repoRoot, { root: repoRoot, entries });
|
|
508
|
-
}
|
|
880
|
+
catch (err) {
|
|
881
|
+
console.error(err instanceof Error ? err.message : String(err));
|
|
882
|
+
process.exitCode = 1;
|
|
509
883
|
}
|
|
510
|
-
}
|
|
884
|
+
};
|
|
511
885
|
}
|
|
512
|
-
|
|
513
|
-
|
|
514
|
-
|
|
515
|
-
* Converts backslashes to forward slashes in paths without other transformations.
|
|
516
|
-
*/
|
|
517
|
-
/**
|
|
518
|
-
* Normalize path separators: replace backslashes with forward slashes.
|
|
519
|
-
*
|
|
520
|
-
* Does NOT lowercase or strip drive letters — use `normalizePath()` if you need that.
|
|
521
|
-
*
|
|
522
|
-
* @param path - The path string to normalize.
|
|
523
|
-
* @returns The path with forward slashes.
|
|
524
|
-
*/
|
|
525
|
-
function normalizeSlashes(path) {
|
|
526
|
-
return path.replace(/\\/g, '/');
|
|
886
|
+
/** Print JSON response to stdout. */
|
|
887
|
+
function printJson(data) {
|
|
888
|
+
console.log(JSON.stringify(data, null, 2));
|
|
527
889
|
}
|
|
528
890
|
|
|
529
891
|
/**
|
|
530
|
-
* @module
|
|
531
|
-
*
|
|
532
|
-
*/
|
|
533
|
-
/**
|
|
534
|
-
* Check whether a file path matches the watched scope.
|
|
535
|
-
* A path is watched if it matches at least one `paths` glob
|
|
536
|
-
* AND does not match any `ignored` glob. Mirrors chokidar's
|
|
537
|
-
* inclusion/exclusion logic.
|
|
892
|
+
* @module cli/jeeves-watcher/configCommands
|
|
893
|
+
* Re-wires core's `config` and `config apply` CLI commands through the shared API client, adding `--api-key` so they send the bearer key. Reads patches from a file or stdin.
|
|
538
894
|
*
|
|
539
|
-
* @
|
|
540
|
-
*
|
|
541
|
-
*
|
|
542
|
-
*
|
|
895
|
+
* @remarks
|
|
896
|
+
* Core's `createServiceCli` registers both commands with an HTTP helper that
|
|
897
|
+
* cannot attach headers. This module keeps their names, arguments, options and
|
|
898
|
+
* output, adds `--api-key` and replaces only the action handlers. When the
|
|
899
|
+
* commands are absent (e.g. a bare test program) it does nothing.
|
|
543
900
|
*/
|
|
544
|
-
|
|
545
|
-
|
|
546
|
-
|
|
547
|
-
|
|
548
|
-
|
|
549
|
-
|
|
550
|
-
|
|
551
|
-
|
|
552
|
-
|
|
553
|
-
|
|
554
|
-
|
|
555
|
-
return
|
|
901
|
+
/** Host core's config commands always address. */
|
|
902
|
+
const CONFIG_HOST = API_DEFAULTS.host;
|
|
903
|
+
/** Parsed options of the re-wired commands; Zod is the source of truth. */
|
|
904
|
+
const configOptsSchema = z.object({
|
|
905
|
+
port: z.string(),
|
|
906
|
+
apiKey: z.string().optional(),
|
|
907
|
+
file: z.string().optional(),
|
|
908
|
+
replace: z.boolean().optional(),
|
|
909
|
+
});
|
|
910
|
+
/** Find a direct subcommand by name. */
|
|
911
|
+
function findCommand(parent, name) {
|
|
912
|
+
return parent.commands.find((cmd) => cmd.name() === name);
|
|
913
|
+
}
|
|
914
|
+
/** Add `--api-key` to a command (no default, so it never shows in help). */
|
|
915
|
+
function addApiKeyOption(cmd) {
|
|
916
|
+
cmd.option('--api-key <key>', `API bearer key (defaults to the ${API_KEY_ENV_VAR} environment variable)`);
|
|
917
|
+
}
|
|
918
|
+
/** Print an error with core's prefix and set exit code 1. */
|
|
919
|
+
function reportFailure(prefix, err) {
|
|
920
|
+
console.error(`${prefix}: ${err instanceof Error ? err.message : String(err)}`);
|
|
921
|
+
process.exitCode = 1;
|
|
556
922
|
}
|
|
557
|
-
|
|
558
923
|
/**
|
|
559
|
-
*
|
|
924
|
+
* Read a config patch from `file`, else from stdin (empty input = `{}`).
|
|
560
925
|
*
|
|
561
|
-
*
|
|
926
|
+
* @throws With a one-line message when the file cannot be read, the input
|
|
927
|
+
* is not JSON, or it is not a JSON object.
|
|
562
928
|
*/
|
|
563
|
-
|
|
564
|
-
|
|
565
|
-
|
|
566
|
-
*
|
|
567
|
-
* @param error - The caught value (may not be an Error).
|
|
568
|
-
* @returns A proper Error instance.
|
|
569
|
-
*/
|
|
570
|
-
function normalizeError(error) {
|
|
571
|
-
if (error instanceof Error)
|
|
572
|
-
return error;
|
|
573
|
-
if (typeof error === 'string')
|
|
574
|
-
return new Error(error);
|
|
575
|
-
const message = typeof error === 'object' && error !== null && 'message' in error
|
|
576
|
-
? String(error.message)
|
|
577
|
-
: String(error);
|
|
578
|
-
const normalized = new Error(message);
|
|
579
|
-
normalized.cause = error;
|
|
580
|
-
return normalized;
|
|
581
|
-
}
|
|
582
|
-
|
|
583
|
-
/**
|
|
584
|
-
* @module util/sleep
|
|
585
|
-
* Abort-aware async sleep.
|
|
586
|
-
*/
|
|
587
|
-
/**
|
|
588
|
-
* Sleep for a given duration with optional abort support.
|
|
589
|
-
*
|
|
590
|
-
* @param ms - Duration in milliseconds.
|
|
591
|
-
* @param signal - Optional abort signal.
|
|
592
|
-
*/
|
|
593
|
-
function sleep(ms, signal) {
|
|
594
|
-
if (ms <= 0)
|
|
595
|
-
return Promise.resolve();
|
|
596
|
-
return new Promise((resolve, reject) => {
|
|
597
|
-
const timer = setTimeout(() => {
|
|
598
|
-
cleanup();
|
|
599
|
-
resolve();
|
|
600
|
-
}, ms);
|
|
601
|
-
const onAbort = () => {
|
|
602
|
-
cleanup();
|
|
603
|
-
reject(new Error('Sleep aborted'));
|
|
604
|
-
};
|
|
605
|
-
const cleanup = () => {
|
|
606
|
-
clearTimeout(timer);
|
|
607
|
-
if (signal)
|
|
608
|
-
signal.removeEventListener('abort', onAbort);
|
|
609
|
-
};
|
|
610
|
-
if (signal) {
|
|
611
|
-
if (signal.aborted) {
|
|
612
|
-
onAbort();
|
|
613
|
-
return;
|
|
614
|
-
}
|
|
615
|
-
signal.addEventListener('abort', onAbort, { once: true });
|
|
616
|
-
}
|
|
617
|
-
});
|
|
618
|
-
}
|
|
619
|
-
|
|
620
|
-
/**
|
|
621
|
-
* @module util/retry
|
|
622
|
-
* Small async retry helper with exponential backoff. Side effects: sleeps between attempts; can invoke onRetry callback for logging.
|
|
623
|
-
*/
|
|
624
|
-
function computeDelayMs(attempt, baseDelayMs, maxDelayMs, jitter = 0) {
|
|
625
|
-
const exp = Math.max(0, attempt - 1);
|
|
626
|
-
const raw = Math.min(maxDelayMs, baseDelayMs * 2 ** exp);
|
|
627
|
-
const factor = jitter > 0 ? 1 + Math.random() * jitter : 1;
|
|
628
|
-
return Math.round(raw * factor);
|
|
629
|
-
}
|
|
630
|
-
/**
|
|
631
|
-
* Retry an async operation using exponential backoff.
|
|
632
|
-
*
|
|
633
|
-
* @param fn - Operation to execute.
|
|
634
|
-
* @param options - Retry policy.
|
|
635
|
-
* @returns The operation result.
|
|
636
|
-
*/
|
|
637
|
-
async function retry(fn, options) {
|
|
638
|
-
const attempts = Math.max(1, options.attempts);
|
|
639
|
-
const shouldRetry = options.shouldRetry ?? (() => true);
|
|
640
|
-
let lastError;
|
|
641
|
-
for (let attempt = 1; attempt <= attempts; attempt++) {
|
|
929
|
+
async function readPatch(file) {
|
|
930
|
+
let input;
|
|
931
|
+
if (file) {
|
|
642
932
|
try {
|
|
643
|
-
|
|
933
|
+
input = readFileSync(file, 'utf-8');
|
|
644
934
|
}
|
|
645
|
-
catch (
|
|
646
|
-
|
|
647
|
-
|
|
648
|
-
if (isLast || !shouldRetry(error))
|
|
649
|
-
break;
|
|
650
|
-
const delayMs = computeDelayMs(attempt, options.baseDelayMs, options.maxDelayMs, options.jitter);
|
|
651
|
-
options.onRetry?.({ attempt, attempts, delayMs, error });
|
|
652
|
-
await sleep(delayMs, options.signal);
|
|
935
|
+
catch (err) {
|
|
936
|
+
const { code } = err;
|
|
937
|
+
throw new Error(`cannot read patch file ${file}${typeof code === 'string' ? ` (${code})` : ''}`, { cause: err });
|
|
653
938
|
}
|
|
654
939
|
}
|
|
655
|
-
|
|
656
|
-
|
|
657
|
-
|
|
658
|
-
|
|
659
|
-
|
|
660
|
-
* Shared git execution utilities: the pinned argv prefix, timeouts, error
|
|
661
|
-
* field extraction and classification, and path comparison helpers.
|
|
662
|
-
*/
|
|
663
|
-
/** Promisified execFile for git commands. */
|
|
664
|
-
const execFileAsync = promisify(execFile);
|
|
665
|
-
/**
|
|
666
|
-
* Leading git arguments applied to every VCS-owned git invocation.
|
|
667
|
-
*
|
|
668
|
-
* `-c core.longpaths=true` lets Windows checkouts handle paths beyond the
|
|
669
|
-
* legacy MAX_PATH limit (260 chars) without requiring a machine-wide
|
|
670
|
-
* `core.longpaths` git config or registry change. See #249.
|
|
671
|
-
*/
|
|
672
|
-
const GIT_BASE_ARGS = ['-c', 'core.longpaths=true'];
|
|
673
|
-
/**
|
|
674
|
-
* Build a git argv array with {@link GIT_BASE_ARGS} prefixed.
|
|
675
|
-
*
|
|
676
|
-
* @param args - The subcommand and its arguments (e.g. `'commit', '-m', msg`).
|
|
677
|
-
* @returns The full argv array, base args first.
|
|
678
|
-
*/
|
|
679
|
-
function gitArgs(...args) {
|
|
680
|
-
return [...GIT_BASE_ARGS, ...args];
|
|
681
|
-
}
|
|
682
|
-
/** Standard timeout (ms) for most git operations. */
|
|
683
|
-
const GIT_TIMEOUT_STANDARD = 30_000;
|
|
684
|
-
/** Extended timeout (ms) for cherry-pick operations. */
|
|
685
|
-
const GIT_TIMEOUT_CHERRY_PICK = 120_000;
|
|
686
|
-
/** Timeout (ms) for push operations. */
|
|
687
|
-
const GIT_TIMEOUT_PUSH = 60_000;
|
|
688
|
-
/**
|
|
689
|
-
* Extract string fields from a child-process error.
|
|
690
|
-
* Node's execFile wraps failures in an Error with `stderr`, `stdout`, and
|
|
691
|
-
* `code` properties that aren't part of the Error type.
|
|
692
|
-
*/
|
|
693
|
-
function getExecErrorFields(error) {
|
|
694
|
-
if (!(error instanceof Error)) {
|
|
695
|
-
return { message: String(error), stderr: '', stdout: '' };
|
|
940
|
+
else {
|
|
941
|
+
const chunks = [];
|
|
942
|
+
for await (const chunk of process.stdin)
|
|
943
|
+
chunks.push(chunk);
|
|
944
|
+
input = Buffer.concat(chunks).toString('utf-8');
|
|
696
945
|
}
|
|
697
|
-
|
|
698
|
-
|
|
699
|
-
|
|
700
|
-
|
|
701
|
-
|
|
702
|
-
/**
|
|
703
|
-
* Read the `code` property of an unknown rejection value, if present.
|
|
704
|
-
* execFile rejects with the child's numeric exit status in `code`; fs calls
|
|
705
|
-
* reject with a string errno code (e.g. `'ENOENT'`).
|
|
706
|
-
*
|
|
707
|
-
* @param error - The caught value.
|
|
708
|
-
* @returns The code, or undefined if the value has none.
|
|
709
|
-
*/
|
|
710
|
-
function getErrorCode(error) {
|
|
711
|
-
if (typeof error !== 'object' || error === null || !('code' in error)) {
|
|
712
|
-
return undefined;
|
|
946
|
+
if (!input.trim())
|
|
947
|
+
return {};
|
|
948
|
+
let patch;
|
|
949
|
+
try {
|
|
950
|
+
patch = JSON.parse(input);
|
|
713
951
|
}
|
|
714
|
-
|
|
715
|
-
|
|
716
|
-
? code
|
|
717
|
-
: undefined;
|
|
718
|
-
}
|
|
719
|
-
/**
|
|
720
|
-
* Check whether an error is caused by `index.lock` contention.
|
|
721
|
-
*
|
|
722
|
-
* Requires the literal `index.lock` in the message or stderr: a bare
|
|
723
|
-
* `EEXIST` (any file that already exists) is not lock contention and must
|
|
724
|
-
* not be retried as if it were. Only Error instances qualify.
|
|
725
|
-
*
|
|
726
|
-
* @param error - The caught value.
|
|
727
|
-
* @returns true if the error names `index.lock`.
|
|
728
|
-
*/
|
|
729
|
-
function isIndexLockError(error) {
|
|
730
|
-
if (!(error instanceof Error))
|
|
731
|
-
return false;
|
|
732
|
-
const { message, stderr } = getExecErrorFields(error);
|
|
733
|
-
return message.includes('index.lock') || stderr.includes('index.lock');
|
|
734
|
-
}
|
|
735
|
-
/**
|
|
736
|
-
* Normalize a path for case-insensitive comparison on Windows.
|
|
737
|
-
* On Windows, lowercases the entire path; on other platforms, returns as-is.
|
|
738
|
-
*
|
|
739
|
-
* @param p - The path string to normalize.
|
|
740
|
-
* @param platform - The platform to check against (default: `process.platform`).
|
|
741
|
-
* @returns The normalized path.
|
|
742
|
-
*/
|
|
743
|
-
function normalizePathCase(p, platform = process.platform) {
|
|
744
|
-
return platform === 'win32' ? p.toLowerCase() : p;
|
|
745
|
-
}
|
|
746
|
-
/**
|
|
747
|
-
* Build a comparison key for an absolute path: resolved, forward slashes,
|
|
748
|
-
* and lowercased on Windows.
|
|
749
|
-
*
|
|
750
|
-
* @param p - The path.
|
|
751
|
-
* @param platform - The platform to check against (default: `process.platform`).
|
|
752
|
-
* @returns The comparison key.
|
|
753
|
-
*/
|
|
754
|
-
function pathKey(p, platform = process.platform) {
|
|
755
|
-
return normalizePathCase(normalizeSlashes(resolve(p)), platform);
|
|
756
|
-
}
|
|
757
|
-
/**
|
|
758
|
-
* Find the longest-prefix-match root for a normalized path.
|
|
759
|
-
* Roots must be sorted longest-first for correct nested matching.
|
|
760
|
-
*
|
|
761
|
-
* @param roots - Root paths sorted longest-first.
|
|
762
|
-
* @param normalizedPath - Normalized absolute path (forward slashes).
|
|
763
|
-
* @param platform - The platform to check against (default: `process.platform`).
|
|
764
|
-
* @returns The matching root, or undefined.
|
|
765
|
-
*/
|
|
766
|
-
function findRootForPath(roots, normalizedPath, platform = process.platform) {
|
|
767
|
-
const comparePath = normalizePathCase(normalizedPath, platform);
|
|
768
|
-
for (const root of roots) {
|
|
769
|
-
const compareRoot = normalizePathCase(root, platform);
|
|
770
|
-
if (comparePath === compareRoot ||
|
|
771
|
-
comparePath.startsWith(compareRoot + '/')) {
|
|
772
|
-
return root;
|
|
773
|
-
}
|
|
952
|
+
catch (err) {
|
|
953
|
+
throw new Error(`patch is not valid JSON (${err instanceof Error ? err.message : String(err)})`, { cause: err });
|
|
774
954
|
}
|
|
775
|
-
|
|
955
|
+
if (!isPlainObject$1(patch))
|
|
956
|
+
throw new Error('patch must be a JSON object');
|
|
957
|
+
return patch;
|
|
776
958
|
}
|
|
777
|
-
|
|
778
959
|
/**
|
|
779
|
-
*
|
|
960
|
+
* Make core's `config` and `config apply` commands send the bearer key.
|
|
780
961
|
*
|
|
781
|
-
*
|
|
782
|
-
* to scan the directory tree in integration tests.
|
|
962
|
+
* @param program - The service CLI program, after core registered its commands.
|
|
783
963
|
*/
|
|
784
|
-
function
|
|
785
|
-
const
|
|
786
|
-
|
|
787
|
-
if (globIdx === -1)
|
|
788
|
-
return resolve(pattern);
|
|
789
|
-
const prefix = normalised.slice(0, globIdx);
|
|
790
|
-
// If prefix ends mid-segment, dirname to get a real directory
|
|
791
|
-
const base = prefix.endsWith('/') ? prefix.slice(0, -1) : dirname(prefix);
|
|
792
|
-
return resolve(base);
|
|
793
|
-
}
|
|
794
|
-
async function* walk(dir) {
|
|
795
|
-
let entries;
|
|
796
|
-
try {
|
|
797
|
-
const dirents = await readdir(dir, { withFileTypes: true });
|
|
798
|
-
entries = dirents.map((d) => ({
|
|
799
|
-
name: d.name,
|
|
800
|
-
isDirectory: d.isDirectory(),
|
|
801
|
-
}));
|
|
802
|
-
}
|
|
803
|
-
catch {
|
|
964
|
+
function authenticateConfigCommands(program) {
|
|
965
|
+
const configCmd = findCommand(program, 'config');
|
|
966
|
+
if (!configCmd)
|
|
804
967
|
return;
|
|
805
|
-
|
|
806
|
-
|
|
807
|
-
const
|
|
808
|
-
|
|
809
|
-
|
|
968
|
+
addApiKeyOption(configCmd);
|
|
969
|
+
configCmd.action(async () => {
|
|
970
|
+
const opts = configOptsSchema.parse(configCmd.opts());
|
|
971
|
+
const jsonpath = configCmd.args[0];
|
|
972
|
+
const qs = jsonpath ? `?path=${encodeURIComponent(jsonpath)}` : '';
|
|
973
|
+
try {
|
|
974
|
+
const client = apiClient({ ...opts, host: CONFIG_HOST });
|
|
975
|
+
printJson(await client.get(`${getEndpoint('config').path}${qs}`));
|
|
810
976
|
}
|
|
811
|
-
|
|
812
|
-
|
|
813
|
-
try {
|
|
814
|
-
const st = await stat(full);
|
|
815
|
-
if (st.isFile())
|
|
816
|
-
yield full;
|
|
817
|
-
}
|
|
818
|
-
catch {
|
|
819
|
-
// ignore
|
|
820
|
-
}
|
|
977
|
+
catch (err) {
|
|
978
|
+
reportFailure('Config query failed', err);
|
|
821
979
|
}
|
|
822
|
-
}
|
|
823
|
-
|
|
824
|
-
|
|
825
|
-
|
|
826
|
-
|
|
827
|
-
|
|
828
|
-
|
|
829
|
-
|
|
830
|
-
|
|
831
|
-
|
|
832
|
-
|
|
833
|
-
|
|
834
|
-
|
|
835
|
-
|
|
836
|
-
|
|
837
|
-
|
|
838
|
-
|
|
839
|
-
for (const base of bases) {
|
|
840
|
-
for await (const file of walk(base)) {
|
|
841
|
-
const rel = normalizeSlashes(file);
|
|
842
|
-
if (ignore(rel))
|
|
843
|
-
continue;
|
|
844
|
-
if (!match(rel))
|
|
845
|
-
continue;
|
|
846
|
-
if (extraFilter && !extraFilter(rel))
|
|
847
|
-
continue;
|
|
848
|
-
if (isGitignored?.(file))
|
|
849
|
-
continue;
|
|
850
|
-
seen.add(file);
|
|
980
|
+
});
|
|
981
|
+
const applyCmd = findCommand(configCmd, 'apply');
|
|
982
|
+
if (!applyCmd)
|
|
983
|
+
return;
|
|
984
|
+
addApiKeyOption(applyCmd);
|
|
985
|
+
applyCmd.action(async () => {
|
|
986
|
+
const opts = configOptsSchema.parse(applyCmd.opts());
|
|
987
|
+
try {
|
|
988
|
+
const patch = await readPatch(opts.file);
|
|
989
|
+
const body = { patch };
|
|
990
|
+
if (opts.replace)
|
|
991
|
+
body.replace = true;
|
|
992
|
+
const client = apiClient({ ...opts, host: CONFIG_HOST });
|
|
993
|
+
printJson(await client.post(getEndpoint('configApply').path, body));
|
|
994
|
+
}
|
|
995
|
+
catch (err) {
|
|
996
|
+
reportFailure('Config apply failed', err);
|
|
851
997
|
}
|
|
852
|
-
}
|
|
853
|
-
return Array.from(seen);
|
|
854
|
-
}
|
|
855
|
-
/**
|
|
856
|
-
* List files matching a set of globs, with optional ignore globs and gitignore filter.
|
|
857
|
-
*
|
|
858
|
-
* @param patterns - Glob patterns to match.
|
|
859
|
-
* @param ignored - Glob patterns to exclude (optional).
|
|
860
|
-
* @param isGitignored - Optional callback to check gitignore status per file.
|
|
861
|
-
*/
|
|
862
|
-
async function listFilesFromGlobs(patterns, ignored = [], isGitignored) {
|
|
863
|
-
return walkAndFilter({ patterns, ignored, isGitignored });
|
|
864
|
-
}
|
|
865
|
-
/**
|
|
866
|
-
* List files from watch root base directories, applying watch extension globs,
|
|
867
|
-
* ignored globs, gitignore, and an additional caller-provided glob intersection filter.
|
|
868
|
-
*
|
|
869
|
-
* Unlike `listFilesFromGlobs` (which derives walk roots from the glob patterns themselves),
|
|
870
|
-
* this function always walks from watch root base directories and applies the caller's
|
|
871
|
-
* globs as an additional intersection filter within the watched universe.
|
|
872
|
-
*
|
|
873
|
-
* @param watchPatterns - The configured watch path globs (defines the watched universe).
|
|
874
|
-
* @param ignored - Glob patterns to exclude.
|
|
875
|
-
* @param callerGlobs - Additional globs to intersect with. Only files matching both
|
|
876
|
-
* the watch patterns AND the caller globs are returned.
|
|
877
|
-
* @param isGitignored - Optional callback to check gitignore status per file.
|
|
878
|
-
*/
|
|
879
|
-
async function listFilesFromWatchRoots(watchPatterns, ignored, callerGlobs, isGitignored) {
|
|
880
|
-
const normCaller = callerGlobs.map((p) => normalizeSlashes(p));
|
|
881
|
-
const matchCaller = picomatch(normCaller, { dot: true, nocase: true });
|
|
882
|
-
return walkAndFilter({
|
|
883
|
-
patterns: watchPatterns,
|
|
884
|
-
ignored,
|
|
885
|
-
isGitignored,
|
|
886
|
-
extraFilter: (normalizedPath) => matchCaller(normalizedPath),
|
|
887
998
|
});
|
|
888
999
|
}
|
|
889
|
-
/**
|
|
890
|
-
* Get the base directories of the configured watch paths.
|
|
891
|
-
*/
|
|
892
|
-
function getWatchRootBases(watchPatterns) {
|
|
893
|
-
return Array.from(new Set(watchPatterns.map(globBase)));
|
|
894
|
-
}
|
|
895
1000
|
|
|
896
1001
|
/**
|
|
897
|
-
* @module
|
|
898
|
-
*
|
|
899
|
-
* Shared helper for processing all files matching configured globs.
|
|
1002
|
+
* @module cli/jeeves-watcher/helperFormat
|
|
1003
|
+
* Pure formatting for the `helpers` CLI command: renders JsonMap and Handlebars helper namespaces from GET /config query results as text.
|
|
900
1004
|
*/
|
|
901
|
-
/** Default concurrency limit for reindex operations. */
|
|
902
|
-
const DEFAULT_REINDEX_CONCURRENCY = 50;
|
|
903
1005
|
/**
|
|
904
|
-
*
|
|
1006
|
+
* Format a helpers section for display. Pure.
|
|
905
1007
|
*
|
|
906
|
-
* @param
|
|
907
|
-
* @param
|
|
908
|
-
* @
|
|
909
|
-
* @param method - The processor method to call ('processFile' or 'processRulesUpdate').
|
|
910
|
-
* @param concurrency - Maximum concurrent file operations (default 50).
|
|
911
|
-
* @param callbacks - Optional progress tracking callbacks.
|
|
912
|
-
* @param isGitignored - Optional callback to check gitignore status per file.
|
|
913
|
-
* @param preComputedFiles - Optional pre-computed file list (skips listFilesFromGlobs).
|
|
914
|
-
* @returns The number of files processed.
|
|
915
|
-
*/
|
|
916
|
-
async function processAllFiles(watchPaths, ignoredPaths, processor, method, concurrency = DEFAULT_REINDEX_CONCURRENCY, callbacks, isGitignored, preComputedFiles) {
|
|
917
|
-
const files = preComputedFiles ??
|
|
918
|
-
(await listFilesFromGlobs(watchPaths, ignoredPaths, isGitignored));
|
|
919
|
-
callbacks?.onTotal?.(files.length);
|
|
920
|
-
await parallel(concurrency, files, async (file) => {
|
|
921
|
-
await processor[method](file);
|
|
922
|
-
callbacks?.onFileProcessed?.();
|
|
923
|
-
});
|
|
924
|
-
return files.length;
|
|
925
|
-
}
|
|
926
|
-
|
|
927
|
-
/**
|
|
928
|
-
* @module api/executeReindex
|
|
929
|
-
* Shared reindex execution logic used by both the reindex handler and the triggerReindex lambda.
|
|
930
|
-
*/
|
|
931
|
-
/** Ordered list of valid reindex scopes for validation. */
|
|
932
|
-
const VALID_REINDEX_SCOPES = [
|
|
933
|
-
'issues',
|
|
934
|
-
'full',
|
|
935
|
-
'rules',
|
|
936
|
-
'path',
|
|
937
|
-
'prune',
|
|
938
|
-
];
|
|
939
|
-
/**
|
|
940
|
-
* Valid scopes for config-watch auto-trigger.
|
|
941
|
-
* `prune` is excluded - auto-pruning on config change is dangerous.
|
|
1008
|
+
* @param title - Section heading.
|
|
1009
|
+
* @param data - Helper namespaces keyed by name.
|
|
1010
|
+
* @returns The rendered section, or `''` when there are no helpers.
|
|
942
1011
|
*/
|
|
943
|
-
|
|
944
|
-
|
|
945
|
-
|
|
946
|
-
|
|
947
|
-
]
|
|
948
|
-
|
|
949
|
-
|
|
950
|
-
await retry(async () => {
|
|
951
|
-
const response = await fetch(url, {
|
|
952
|
-
method: 'POST',
|
|
953
|
-
headers: { 'Content-Type': 'application/json' },
|
|
954
|
-
body: JSON.stringify(payload),
|
|
955
|
-
});
|
|
956
|
-
if (!response.ok) {
|
|
957
|
-
throw new Error(`Non-OK response: ${String(response.status)}`);
|
|
1012
|
+
function formatHelperSection(title, data) {
|
|
1013
|
+
if (!data || Object.keys(data).length === 0)
|
|
1014
|
+
return '';
|
|
1015
|
+
const lines = [`${title}:`];
|
|
1016
|
+
for (const [namespace, entry] of Object.entries(data)) {
|
|
1017
|
+
if (entry.description) {
|
|
1018
|
+
lines.push(` [${namespace}] ${entry.description}`);
|
|
958
1019
|
}
|
|
959
|
-
|
|
960
|
-
|
|
961
|
-
|
|
962
|
-
|
|
963
|
-
onRetry: ({ attempt, error }) => {
|
|
964
|
-
logger.warn({ attempt, err: normalizeError(error), url }, 'Reindex callback failed; will retry');
|
|
965
|
-
},
|
|
966
|
-
});
|
|
967
|
-
}
|
|
968
|
-
/**
|
|
969
|
-
* Group file paths by their watch root prefix.
|
|
970
|
-
* Each file is matched against `watchPaths` and grouped under the first matching root.
|
|
971
|
-
* Files that don't match any root are grouped under `'(unmatched)'`.
|
|
972
|
-
*/
|
|
973
|
-
function groupByRoot(filePaths, watchPaths) {
|
|
974
|
-
const byRoot = {};
|
|
975
|
-
for (const fp of filePaths) {
|
|
976
|
-
const normalised = normalizeSlashes(fp);
|
|
977
|
-
let matched = false;
|
|
978
|
-
for (const root of watchPaths) {
|
|
979
|
-
const rootNorm = normalizeSlashes(root).replace(/\/?\*\*.*$/, '');
|
|
980
|
-
if (normalised.startsWith(rootNorm)) {
|
|
981
|
-
byRoot[rootNorm] = (byRoot[rootNorm] ?? 0) + 1;
|
|
982
|
-
matched = true;
|
|
983
|
-
break;
|
|
1020
|
+
if (entry.exports) {
|
|
1021
|
+
for (const [name, desc] of Object.entries(entry.exports)) {
|
|
1022
|
+
const descPart = desc ? ` - ${desc}` : '';
|
|
1023
|
+
lines.push(` ${name.padEnd(40)}${descPart}`);
|
|
984
1024
|
}
|
|
985
1025
|
}
|
|
986
|
-
|
|
987
|
-
|
|
1026
|
+
else {
|
|
1027
|
+
lines.push(` (no exports introspected)`);
|
|
988
1028
|
}
|
|
989
1029
|
}
|
|
990
|
-
return
|
|
1030
|
+
return lines.join('\n');
|
|
991
1031
|
}
|
|
992
|
-
|
|
993
|
-
const PRUNE_SCROLL_PAGE_SIZE = 500;
|
|
994
|
-
/** Max retry attempts for scroll page failures. */
|
|
995
|
-
const SCROLL_RETRY_ATTEMPTS = 5;
|
|
996
|
-
/** Base delay for scroll retry backoff (ms). */
|
|
997
|
-
const SCROLL_RETRY_BASE_DELAY_MS = 1000;
|
|
1032
|
+
|
|
998
1033
|
/**
|
|
999
|
-
*
|
|
1000
|
-
*
|
|
1034
|
+
* @module cli/jeeves-watcher/metadataArgs
|
|
1035
|
+
* Parses the `enrich` command's --json and --key options into a metadata object; reports validation errors on stderr and sets the exit code.
|
|
1001
1036
|
*/
|
|
1002
|
-
|
|
1003
|
-
|
|
1004
|
-
|
|
1005
|
-
try {
|
|
1006
|
-
const page = await vectorStore.scrollPage(undefined, PRUNE_SCROLL_PAGE_SIZE, cursor, ['file_path']);
|
|
1007
|
-
return { points: page.points, nextCursor: page.nextCursor };
|
|
1008
|
-
}
|
|
1009
|
-
catch (error) {
|
|
1010
|
-
lastError = error;
|
|
1011
|
-
if (attempt < SCROLL_RETRY_ATTEMPTS) {
|
|
1012
|
-
const delay = SCROLL_RETRY_BASE_DELAY_MS * Math.pow(2, attempt - 1) +
|
|
1013
|
-
Math.random() * 500;
|
|
1014
|
-
logger.warn({ attempt, err: normalizeError(error) }, `Prune scroll page failed; retrying in ${String(Math.round(delay))}ms`);
|
|
1015
|
-
await new Promise((resolve) => setTimeout(resolve, delay));
|
|
1016
|
-
}
|
|
1017
|
-
}
|
|
1018
|
-
}
|
|
1019
|
-
throw lastError;
|
|
1037
|
+
/** Whether a parsed JSON value is a plain object (not null, an array or a primitive). */
|
|
1038
|
+
function isPlainObject(value) {
|
|
1039
|
+
return typeof value === 'object' && value !== null && !Array.isArray(value);
|
|
1020
1040
|
}
|
|
1021
|
-
/** Concurrency limit for fs.access() disk-existence checks. */
|
|
1022
|
-
const FS_ACCESS_CONCURRENCY = 100;
|
|
1023
1041
|
/**
|
|
1024
|
-
*
|
|
1042
|
+
* Parse `--json` and `--key` options into a metadata object.
|
|
1043
|
+
* `--json` must be a JSON object; null, arrays and primitives are rejected.
|
|
1025
1044
|
*
|
|
1026
|
-
*
|
|
1027
|
-
*
|
|
1028
|
-
*
|
|
1029
|
-
* - `file_path` not in `watchedFiles` → orphan (out of watch scope).
|
|
1030
|
-
* - `file_path` in `watchedFiles` → check disk existence via `fs.access()`.
|
|
1031
|
-
* If missing from disk → orphan (deleted file still in index).
|
|
1032
|
-
* 3. Batch `fs.access()` checks with bounded concurrency.
|
|
1045
|
+
* @param json - Metadata as a JSON string, if given.
|
|
1046
|
+
* @param keys - `key=value` pairs (applied after `json`).
|
|
1047
|
+
* @returns The metadata, or `null` on validation failure (already reported).
|
|
1033
1048
|
*/
|
|
1034
|
-
|
|
1035
|
-
|
|
1036
|
-
if (
|
|
1037
|
-
|
|
1038
|
-
}
|
|
1039
|
-
// Step 1: Build set of watched files from filesystem
|
|
1040
|
-
const isGitignored = createIsGitignored(gitignoreFilter);
|
|
1041
|
-
const watchedFileList = await listFilesFromGlobs(extractWatchPathStrings(config.watch.paths), config.watch.ignored, isGitignored);
|
|
1042
|
-
const watchedFiles = new Set(watchedFileList.map((f) => normalizePathCase(normalizeSlashes(f))));
|
|
1043
|
-
logger.info({ watchedFileCount: watchedFiles.size }, 'Prune: enumerated watched files from filesystem');
|
|
1044
|
-
// Step 2: Scroll Qdrant and classify points
|
|
1045
|
-
const seenFiles = new Set();
|
|
1046
|
-
const orphanedFiles = new Set();
|
|
1047
|
-
const diskCheckFiles = new Set();
|
|
1048
|
-
const orphanedIds = [];
|
|
1049
|
-
// Map from file_path → point IDs, for points needing disk-existence check
|
|
1050
|
-
const diskCheckPointIds = new Map();
|
|
1051
|
-
let totalPoints = 0;
|
|
1052
|
-
let incomplete = false;
|
|
1053
|
-
let cursor;
|
|
1054
|
-
try {
|
|
1055
|
-
let hasMore = true;
|
|
1056
|
-
while (hasMore) {
|
|
1057
|
-
const page = await scrollPageWithRetry(vectorStore, cursor, logger);
|
|
1058
|
-
for (const point of page.points) {
|
|
1059
|
-
totalPoints++;
|
|
1060
|
-
const filePath = point.payload.file_path;
|
|
1061
|
-
if (!filePath) {
|
|
1062
|
-
orphanedIds.push(point.id);
|
|
1063
|
-
continue;
|
|
1064
|
-
}
|
|
1065
|
-
const normalized = normalizePathCase(normalizeSlashes(filePath));
|
|
1066
|
-
if (seenFiles.has(normalized)) {
|
|
1067
|
-
if (orphanedFiles.has(normalized)) {
|
|
1068
|
-
orphanedIds.push(point.id);
|
|
1069
|
-
}
|
|
1070
|
-
else if (diskCheckFiles.has(normalized)) {
|
|
1071
|
-
// Already queued for disk check — track its ID
|
|
1072
|
-
const ids = diskCheckPointIds.get(normalized);
|
|
1073
|
-
if (ids)
|
|
1074
|
-
ids.push(point.id);
|
|
1075
|
-
}
|
|
1076
|
-
continue;
|
|
1077
|
-
}
|
|
1078
|
-
seenFiles.add(normalized);
|
|
1079
|
-
if (!watchedFiles.has(normalized)) {
|
|
1080
|
-
// Not in watch scope → definitely orphaned
|
|
1081
|
-
orphanedFiles.add(normalized);
|
|
1082
|
-
orphanedIds.push(point.id);
|
|
1083
|
-
}
|
|
1084
|
-
else {
|
|
1085
|
-
// In watch scope → needs disk-existence check
|
|
1086
|
-
diskCheckFiles.add(normalized);
|
|
1087
|
-
diskCheckPointIds.set(normalized, [point.id]);
|
|
1088
|
-
}
|
|
1089
|
-
}
|
|
1090
|
-
// Progress logging every 100K points
|
|
1091
|
-
if (totalPoints > 0 && totalPoints % 100000 === 0) {
|
|
1092
|
-
logger.info({ totalPoints, orphanedCount: orphanedIds.length }, 'Prune scroll progress');
|
|
1093
|
-
}
|
|
1094
|
-
if (!page.nextCursor) {
|
|
1095
|
-
hasMore = false;
|
|
1096
|
-
}
|
|
1097
|
-
else {
|
|
1098
|
-
cursor = page.nextCursor;
|
|
1099
|
-
}
|
|
1100
|
-
}
|
|
1101
|
-
}
|
|
1102
|
-
catch (error) {
|
|
1103
|
-
logger.error({ err: normalizeError(error), totalPoints, cursor }, 'Prune scroll failed after retries; returning partial plan');
|
|
1104
|
-
incomplete = true;
|
|
1105
|
-
}
|
|
1106
|
-
// Step 3: Batch fs.access() checks for files in watch scope
|
|
1107
|
-
const filesToCheck = [...diskCheckFiles];
|
|
1108
|
-
await parallel(FS_ACCESS_CONCURRENCY, filesToCheck, async (filePath) => {
|
|
1049
|
+
function parseMetadataArgs(json, keys) {
|
|
1050
|
+
let metadata = {};
|
|
1051
|
+
if (json) {
|
|
1052
|
+
let parsed;
|
|
1109
1053
|
try {
|
|
1110
|
-
|
|
1054
|
+
parsed = JSON.parse(json);
|
|
1111
1055
|
}
|
|
1112
1056
|
catch {
|
|
1113
|
-
|
|
1114
|
-
|
|
1115
|
-
|
|
1116
|
-
if (pointIds) {
|
|
1117
|
-
orphanedIds.push(...pointIds);
|
|
1118
|
-
}
|
|
1119
|
-
}
|
|
1120
|
-
});
|
|
1121
|
-
const orphanedFileList = [...orphanedFiles];
|
|
1122
|
-
const byRoot = groupByRoot(orphanedFileList, extractWatchPathStrings(config.watch.paths));
|
|
1123
|
-
return {
|
|
1124
|
-
plan: {
|
|
1125
|
-
total: totalPoints,
|
|
1126
|
-
toProcess: 0,
|
|
1127
|
-
toDelete: orphanedIds.length,
|
|
1128
|
-
byRoot,
|
|
1129
|
-
...(incomplete ? { incomplete: true } : {}),
|
|
1130
|
-
},
|
|
1131
|
-
orphanedIds,
|
|
1132
|
-
};
|
|
1133
|
-
}
|
|
1134
|
-
/**
|
|
1135
|
-
* Execute a reindex operation: process files, track progress, and fire callback if configured.
|
|
1136
|
-
*
|
|
1137
|
-
* @param deps - Dependencies.
|
|
1138
|
-
* @param scope - Reindex scope: 'issues', 'full', 'rules', 'path', or 'prune'.
|
|
1139
|
-
* @param path - Target path (required when scope is 'path').
|
|
1140
|
-
* @param dryRun - When true, compute and return the plan without executing.
|
|
1141
|
-
* @returns The reindex result including the blast area plan.
|
|
1142
|
-
*/
|
|
1143
|
-
async function executeReindex(deps, scope, path, dryRun = false) {
|
|
1144
|
-
const { config, processor, logger, reindexTracker, valuesManager, gitignoreFilter, } = deps;
|
|
1145
|
-
const isGitignored = createIsGitignored(gitignoreFilter);
|
|
1146
|
-
if (!VALID_REINDEX_SCOPES.includes(scope)) {
|
|
1147
|
-
throw new Error(`Invalid reindex scope: "${scope}". Valid scopes: ${VALID_REINDEX_SCOPES.join(', ')}`);
|
|
1148
|
-
}
|
|
1149
|
-
const pathArray = path
|
|
1150
|
-
? Array.isArray(path)
|
|
1151
|
-
? path
|
|
1152
|
-
: [path]
|
|
1153
|
-
: undefined;
|
|
1154
|
-
if (scope === 'path' && (!pathArray || pathArray.length === 0)) {
|
|
1155
|
-
throw new Error('The "path" parameter is required when scope is "path".');
|
|
1156
|
-
}
|
|
1157
|
-
if (scope === 'prune' && !deps.vectorStore) {
|
|
1158
|
-
throw new Error('vectorStore is required for prune scope.');
|
|
1159
|
-
}
|
|
1160
|
-
// Compute plan before starting async work
|
|
1161
|
-
let plan;
|
|
1162
|
-
if (scope === 'prune') {
|
|
1163
|
-
deps.queue?.pause();
|
|
1164
|
-
await deps.queue?.waitForIdleWorkers();
|
|
1165
|
-
let pruneResult;
|
|
1166
|
-
try {
|
|
1167
|
-
pruneResult = await computePrunePlan(deps);
|
|
1168
|
-
}
|
|
1169
|
-
catch (err) {
|
|
1170
|
-
deps.queue?.resume();
|
|
1171
|
-
throw err;
|
|
1172
|
-
}
|
|
1173
|
-
plan = pruneResult.plan;
|
|
1174
|
-
if (dryRun) {
|
|
1175
|
-
deps.queue?.resume();
|
|
1176
|
-
return { filesProcessed: 0, durationMs: 0, errors: 0, plan };
|
|
1177
|
-
}
|
|
1178
|
-
// Abort live prune when plan is incomplete — never silently delete partial data
|
|
1179
|
-
if (plan.incomplete) {
|
|
1180
|
-
deps.queue?.resume();
|
|
1181
|
-
logger.error({ totalPoints: plan.total, pointsToDelete: plan.toDelete }, 'Aborting live prune — scroll was incomplete; partial deletes are unsafe');
|
|
1182
|
-
return { filesProcessed: 0, durationMs: 0, errors: 1, plan };
|
|
1183
|
-
}
|
|
1184
|
-
// Execute prune
|
|
1185
|
-
reindexTracker?.start(scope);
|
|
1186
|
-
const startTime = Date.now();
|
|
1187
|
-
try {
|
|
1188
|
-
if (pruneResult.orphanedIds.length > 0) {
|
|
1189
|
-
const batchSize = 500;
|
|
1190
|
-
for (let i = 0; i < pruneResult.orphanedIds.length; i += batchSize) {
|
|
1191
|
-
const batch = pruneResult.orphanedIds.slice(i, i + batchSize);
|
|
1192
|
-
await deps.vectorStore.delete(batch);
|
|
1193
|
-
}
|
|
1194
|
-
}
|
|
1195
|
-
const durationMs = Date.now() - startTime;
|
|
1196
|
-
logger.info({ scope, pointsDeleted: pruneResult.orphanedIds.length, durationMs }, `Reindex (prune) completed`);
|
|
1197
|
-
reindexTracker?.complete();
|
|
1198
|
-
if (config.reindex?.callbackUrl) {
|
|
1199
|
-
await fireCallback(config.reindex.callbackUrl, {
|
|
1200
|
-
scope,
|
|
1201
|
-
filesProcessed: 0,
|
|
1202
|
-
durationMs,
|
|
1203
|
-
pointsDeleted: pruneResult.orphanedIds.length,
|
|
1204
|
-
errors: [],
|
|
1205
|
-
}, logger);
|
|
1206
|
-
}
|
|
1207
|
-
return {
|
|
1208
|
-
filesProcessed: 0,
|
|
1209
|
-
durationMs,
|
|
1210
|
-
errors: 0,
|
|
1211
|
-
plan,
|
|
1212
|
-
};
|
|
1213
|
-
}
|
|
1214
|
-
catch (error) {
|
|
1215
|
-
const durationMs = Date.now() - startTime;
|
|
1216
|
-
logger.error({ err: normalizeError(error), scope }, 'Prune failed');
|
|
1217
|
-
reindexTracker?.complete();
|
|
1218
|
-
if (config.reindex?.callbackUrl) {
|
|
1219
|
-
await fireCallback(config.reindex.callbackUrl, {
|
|
1220
|
-
scope,
|
|
1221
|
-
filesProcessed: 0,
|
|
1222
|
-
durationMs,
|
|
1223
|
-
errors: [normalizeError(error).message],
|
|
1224
|
-
}, logger);
|
|
1225
|
-
}
|
|
1226
|
-
return { filesProcessed: 0, durationMs, errors: 1, plan };
|
|
1227
|
-
}
|
|
1228
|
-
finally {
|
|
1229
|
-
deps.queue?.resume();
|
|
1230
|
-
}
|
|
1231
|
-
}
|
|
1232
|
-
// Non-prune scopes: compute plan from file lists
|
|
1233
|
-
let fileList;
|
|
1234
|
-
if (scope === 'issues' && deps.issuesManager) {
|
|
1235
|
-
const issues = deps.issuesManager.getAll();
|
|
1236
|
-
fileList = Object.keys(issues);
|
|
1237
|
-
}
|
|
1238
|
-
else if (scope === 'path' && pathArray) {
|
|
1239
|
-
// Path plan computed inline in the execution block below
|
|
1240
|
-
fileList = undefined;
|
|
1241
|
-
}
|
|
1242
|
-
else if (scope === 'rules' && pathArray && pathArray.length > 0) {
|
|
1243
|
-
// Rules scope with path filter: list from watch roots intersected with caller globs
|
|
1244
|
-
fileList = await listFilesFromWatchRoots(extractWatchPathStrings(config.watch.paths), config.watch.ignored, pathArray, isGitignored);
|
|
1245
|
-
}
|
|
1246
|
-
else {
|
|
1247
|
-
// rules or full: list files from globs
|
|
1248
|
-
fileList = await listFilesFromGlobs(extractWatchPathStrings(config.watch.paths), config.watch.ignored, isGitignored);
|
|
1249
|
-
}
|
|
1250
|
-
if (fileList !== undefined) {
|
|
1251
|
-
plan = {
|
|
1252
|
-
total: fileList.length,
|
|
1253
|
-
toProcess: fileList.length,
|
|
1254
|
-
toDelete: 0,
|
|
1255
|
-
byRoot: groupByRoot(fileList, extractWatchPathStrings(config.watch.paths)),
|
|
1256
|
-
};
|
|
1257
|
-
}
|
|
1258
|
-
if (dryRun) {
|
|
1259
|
-
// For path scope without pre-computed plan, compute it now
|
|
1260
|
-
if (plan === undefined && scope === 'path' && pathArray) {
|
|
1261
|
-
const pathFiles = await collectPathFiles(pathArray, config, deps.gitignoreFilter);
|
|
1262
|
-
plan = {
|
|
1263
|
-
total: pathFiles.length,
|
|
1264
|
-
toProcess: pathFiles.length,
|
|
1265
|
-
toDelete: 0,
|
|
1266
|
-
byRoot: groupByRoot(pathFiles, extractWatchPathStrings(config.watch.paths)),
|
|
1267
|
-
};
|
|
1268
|
-
}
|
|
1269
|
-
return { filesProcessed: 0, durationMs: 0, errors: 0, plan };
|
|
1270
|
-
}
|
|
1271
|
-
reindexTracker?.start(scope);
|
|
1272
|
-
const startTime = Date.now();
|
|
1273
|
-
let filesProcessed = 0;
|
|
1274
|
-
let errors = 0;
|
|
1275
|
-
const concurrency = config.reindex?.concurrency ?? 50;
|
|
1276
|
-
try {
|
|
1277
|
-
if (scope === 'full' && valuesManager) {
|
|
1278
|
-
valuesManager.clearAll();
|
|
1279
|
-
}
|
|
1280
|
-
if (scope === 'issues' && fileList) {
|
|
1281
|
-
await parallel(concurrency, fileList, async (filePath) => {
|
|
1282
|
-
try {
|
|
1283
|
-
await processor.processFile(filePath);
|
|
1284
|
-
filesProcessed++;
|
|
1285
|
-
}
|
|
1286
|
-
catch (error) {
|
|
1287
|
-
errors++;
|
|
1288
|
-
logger.warn({ filePath, err: normalizeError(error) }, 'Failed to reprocess issue file');
|
|
1289
|
-
}
|
|
1290
|
-
});
|
|
1291
|
-
}
|
|
1292
|
-
else if (scope === 'rules') {
|
|
1293
|
-
filesProcessed = await processAllFiles(extractWatchPathStrings(config.watch.paths), config.watch.ignored, processor, 'processRulesUpdate', concurrency, {
|
|
1294
|
-
onTotal: (total) => reindexTracker?.setTotal(total),
|
|
1295
|
-
onFileProcessed: () => reindexTracker?.incrementProcessed(),
|
|
1296
|
-
}, isGitignored, fileList);
|
|
1297
|
-
}
|
|
1298
|
-
else if (scope === 'path' && pathArray) {
|
|
1299
|
-
const uniqueFiles = await collectPathFiles(pathArray, config, deps.gitignoreFilter);
|
|
1300
|
-
plan = {
|
|
1301
|
-
total: uniqueFiles.length,
|
|
1302
|
-
toProcess: uniqueFiles.length,
|
|
1303
|
-
toDelete: 0,
|
|
1304
|
-
byRoot: groupByRoot(uniqueFiles, extractWatchPathStrings(config.watch.paths)),
|
|
1305
|
-
};
|
|
1306
|
-
reindexTracker?.setTotal(uniqueFiles.length);
|
|
1307
|
-
await parallel(concurrency, uniqueFiles, async (filePath) => {
|
|
1308
|
-
await processor.processFile(filePath);
|
|
1309
|
-
reindexTracker?.incrementProcessed();
|
|
1310
|
-
filesProcessed++;
|
|
1311
|
-
});
|
|
1312
|
-
}
|
|
1313
|
-
else {
|
|
1314
|
-
// Full reindex
|
|
1315
|
-
filesProcessed = await processAllFiles(extractWatchPathStrings(config.watch.paths), config.watch.ignored, processor, 'processFile', concurrency, {
|
|
1316
|
-
onTotal: (total) => reindexTracker?.setTotal(total),
|
|
1317
|
-
onFileProcessed: () => reindexTracker?.incrementProcessed(),
|
|
1318
|
-
}, isGitignored, fileList);
|
|
1057
|
+
console.error('Invalid JSON:', json);
|
|
1058
|
+
process.exitCode = 1;
|
|
1059
|
+
return null;
|
|
1319
1060
|
}
|
|
1320
|
-
|
|
1321
|
-
|
|
1322
|
-
|
|
1323
|
-
|
|
1324
|
-
await fireCallback(config.reindex.callbackUrl, { scope, filesProcessed, durationMs, errors: [] }, logger);
|
|
1061
|
+
if (!isPlainObject(parsed)) {
|
|
1062
|
+
console.error('JSON metadata must be an object:', json);
|
|
1063
|
+
process.exitCode = 1;
|
|
1064
|
+
return null;
|
|
1325
1065
|
}
|
|
1326
|
-
|
|
1066
|
+
metadata = parsed;
|
|
1327
1067
|
}
|
|
1328
|
-
|
|
1329
|
-
|
|
1330
|
-
|
|
1331
|
-
|
|
1332
|
-
|
|
1333
|
-
|
|
1334
|
-
|
|
1335
|
-
|
|
1336
|
-
|
|
1337
|
-
durationMs,
|
|
1338
|
-
errors: [normalizeError(error).message],
|
|
1339
|
-
}, logger);
|
|
1068
|
+
if (Array.isArray(keys) && keys.length > 0) {
|
|
1069
|
+
for (const pair of keys) {
|
|
1070
|
+
const eqIndex = pair.indexOf('=');
|
|
1071
|
+
if (eqIndex === -1) {
|
|
1072
|
+
console.error(`Invalid key-value pair: ${pair}`);
|
|
1073
|
+
process.exitCode = 1;
|
|
1074
|
+
return null;
|
|
1075
|
+
}
|
|
1076
|
+
metadata[pair.slice(0, eqIndex)] = pair.slice(eqIndex + 1);
|
|
1340
1077
|
}
|
|
1341
|
-
return { filesProcessed: 0, durationMs, errors, plan };
|
|
1342
|
-
}
|
|
1343
|
-
}
|
|
1344
|
-
/**
|
|
1345
|
-
* Collect and deduplicate files for a path-scoped reindex.
|
|
1346
|
-
*
|
|
1347
|
-
* Iterates over each target path, resolves its file list, and returns a
|
|
1348
|
-
* deduplicated array. This eliminates the duplicate iteration + `new Set()`
|
|
1349
|
-
* that previously appeared in both the dryRun plan and execution branches.
|
|
1350
|
-
*/
|
|
1351
|
-
async function collectPathFiles(pathArray, config, gitignoreFilter) {
|
|
1352
|
-
const allFiles = [];
|
|
1353
|
-
for (const p of pathArray) {
|
|
1354
|
-
const files = await getPathFileList(p, config, gitignoreFilter);
|
|
1355
|
-
allFiles.push(...files);
|
|
1356
|
-
}
|
|
1357
|
-
return [...new Set(allFiles)];
|
|
1358
|
-
}
|
|
1359
|
-
/**
|
|
1360
|
-
* Get the file list for a path-scoped reindex (for plan computation).
|
|
1361
|
-
*/
|
|
1362
|
-
async function getPathFileList(targetPath, config, gitignoreFilter) {
|
|
1363
|
-
const watched = isPathWatched(targetPath, extractWatchPathStrings(config.watch.paths), config.watch.ignored);
|
|
1364
|
-
if (!watched) {
|
|
1365
|
-
throw new Error(`Path is outside watch scope: ${targetPath}`);
|
|
1366
|
-
}
|
|
1367
|
-
if (gitignoreFilter?.isIgnored(targetPath)) {
|
|
1368
|
-
throw new Error(`Path is gitignored: ${targetPath}`);
|
|
1369
1078
|
}
|
|
1370
|
-
|
|
1371
|
-
|
|
1372
|
-
|
|
1373
|
-
|
|
1374
|
-
const isGitignored = createIsGitignored(gitignoreFilter);
|
|
1375
|
-
const allWatchedFiles = await listFilesFromGlobs(extractWatchPathStrings(config.watch.paths), config.watch.ignored, isGitignored);
|
|
1376
|
-
const targetAbsPath = resolve(targetPath);
|
|
1377
|
-
return allWatchedFiles.filter((f) => resolve(f).startsWith(targetAbsPath));
|
|
1079
|
+
if (Object.keys(metadata).length === 0) {
|
|
1080
|
+
console.error('No metadata provided. Use --key or --json.');
|
|
1081
|
+
process.exitCode = 1;
|
|
1082
|
+
return null;
|
|
1378
1083
|
}
|
|
1379
|
-
|
|
1084
|
+
return metadata;
|
|
1380
1085
|
}
|
|
1381
1086
|
|
|
1382
1087
|
/**
|
|
1383
1088
|
* @module cli/jeeves-watcher/customCommands
|
|
1384
1089
|
* Domain-specific CLI commands registered via the descriptor's customCliCommands.
|
|
1385
|
-
*
|
|
1090
|
+
* Every API call goes through the shared client (`apiClient`), which sends the bearer key when one is resolved.
|
|
1386
1091
|
*/
|
|
1387
|
-
const DEFAULT_PORT = String(DEFAULT_PORTS.watcher);
|
|
1388
|
-
const DEFAULT_HOST = '127.0.0.1';
|
|
1389
|
-
/** Build the API base URL from host and port options. */
|
|
1390
|
-
function baseUrl(opts) {
|
|
1391
|
-
return `http://${opts.host}:${opts.port}`;
|
|
1392
|
-
}
|
|
1393
|
-
/** Add standard --port and --host options to a command. */
|
|
1394
|
-
function withApiOptions(cmd) {
|
|
1395
|
-
return cmd
|
|
1396
|
-
.option('-p, --port <port>', 'API port', DEFAULT_PORT)
|
|
1397
|
-
.option('-H, --host <host>', 'API host', DEFAULT_HOST);
|
|
1398
|
-
}
|
|
1399
|
-
/** Wrap an async action with standard error handling. */
|
|
1400
|
-
function handleErrors(fn) {
|
|
1401
|
-
return async () => {
|
|
1402
|
-
try {
|
|
1403
|
-
await fn();
|
|
1404
|
-
}
|
|
1405
|
-
catch (err) {
|
|
1406
|
-
console.error(err instanceof Error ? err.message : String(err));
|
|
1407
|
-
process.exitCode = 1;
|
|
1408
|
-
}
|
|
1409
|
-
};
|
|
1410
|
-
}
|
|
1411
|
-
/** Print JSON response to stdout. */
|
|
1412
|
-
function printJson(data) {
|
|
1413
|
-
console.log(JSON.stringify(data, null, 2));
|
|
1414
|
-
}
|
|
1415
1092
|
/**
|
|
1416
1093
|
* Register all domain-specific commands on the given Commander program.
|
|
1417
1094
|
*
|
|
1418
1095
|
* @param program - The Commander program from createServiceCli.
|
|
1419
1096
|
*/
|
|
1420
1097
|
function registerCustomCommands(program) {
|
|
1098
|
+
// Core's `config` / `config apply` call the API too: give them the key.
|
|
1099
|
+
authenticateConfigCommands(program);
|
|
1421
1100
|
// --- search ---
|
|
1422
1101
|
withApiOptions(program
|
|
1423
1102
|
.command('search')
|
|
@@ -1425,7 +1104,7 @@ function registerCustomCommands(program) {
|
|
|
1425
1104
|
.argument('<query>', 'Search query')
|
|
1426
1105
|
.option('-l, --limit <limit>', 'Max results', '10')).action(async (query, opts) => {
|
|
1427
1106
|
await handleErrors(async () => {
|
|
1428
|
-
const result = await
|
|
1107
|
+
const result = await apiClient(opts).post(getEndpoint('search').path, {
|
|
1429
1108
|
query,
|
|
1430
1109
|
limit: Number(opts.limit),
|
|
1431
1110
|
});
|
|
@@ -1443,7 +1122,7 @@ function registerCustomCommands(program) {
|
|
|
1443
1122
|
const metadata = parseMetadataArgs(opts.json, opts.key);
|
|
1444
1123
|
if (!metadata)
|
|
1445
1124
|
return;
|
|
1446
|
-
const result = await
|
|
1125
|
+
const result = await apiClient(opts).post(getEndpoint('metadata').path, {
|
|
1447
1126
|
path,
|
|
1448
1127
|
metadata,
|
|
1449
1128
|
});
|
|
@@ -1461,7 +1140,7 @@ function registerCustomCommands(program) {
|
|
|
1461
1140
|
.option('--count-only', 'Return count only')).action(async (opts) => {
|
|
1462
1141
|
await handleErrors(async () => {
|
|
1463
1142
|
const filterObj = JSON.parse(opts.filter);
|
|
1464
|
-
const result = await
|
|
1143
|
+
const result = await apiClient(opts).post(getEndpoint('scan').path, {
|
|
1465
1144
|
filter: filterObj,
|
|
1466
1145
|
limit: Number(opts.limit),
|
|
1467
1146
|
cursor: opts.cursor,
|
|
@@ -1480,8 +1159,8 @@ function registerCustomCommands(program) {
|
|
|
1480
1159
|
.option('-s, --scope <scope>', 'Reindex scope (issues|full|rules|path|prune)', 'rules')
|
|
1481
1160
|
.option('-t, --path <paths...>', 'Target path(s) for path or rules scope')).action(async (opts) => {
|
|
1482
1161
|
await handleErrors(async () => {
|
|
1483
|
-
if (!
|
|
1484
|
-
console.error(`Invalid scope "${opts.scope}". Must be one of: ${
|
|
1162
|
+
if (!reindexScopeSchema.safeParse(opts.scope).success) {
|
|
1163
|
+
console.error(`Invalid scope "${opts.scope}". Must be one of: ${reindexScopeSchema.options.join(', ')}`);
|
|
1485
1164
|
process.exitCode = 1;
|
|
1486
1165
|
return;
|
|
1487
1166
|
}
|
|
@@ -1489,7 +1168,7 @@ function registerCustomCommands(program) {
|
|
|
1489
1168
|
if (opts.path) {
|
|
1490
1169
|
body.path = opts.path.length === 1 ? opts.path[0] : opts.path;
|
|
1491
1170
|
}
|
|
1492
|
-
const result = await
|
|
1171
|
+
const result = await apiClient(opts).post(getEndpoint('reindex').path, body);
|
|
1493
1172
|
printJson(result);
|
|
1494
1173
|
})();
|
|
1495
1174
|
});
|
|
@@ -1498,7 +1177,7 @@ function registerCustomCommands(program) {
|
|
|
1498
1177
|
.command('rebuild-metadata')
|
|
1499
1178
|
.description('Rebuild metadata store from Qdrant (POST /rebuild-metadata)')).action(async (opts) => {
|
|
1500
1179
|
await handleErrors(async () => {
|
|
1501
|
-
const result = await
|
|
1180
|
+
const result = await apiClient(opts).post(getEndpoint('rebuildMetadata').path, {});
|
|
1502
1181
|
printJson(result);
|
|
1503
1182
|
})();
|
|
1504
1183
|
});
|
|
@@ -1507,19 +1186,18 @@ function registerCustomCommands(program) {
|
|
|
1507
1186
|
.command('issues')
|
|
1508
1187
|
.description('List current processing issues (GET /issues)')).action(async (opts) => {
|
|
1509
1188
|
await handleErrors(async () => {
|
|
1510
|
-
const result = await
|
|
1189
|
+
const result = await apiClient(opts).get(getEndpoint('issues').path);
|
|
1511
1190
|
printJson(result);
|
|
1512
1191
|
})();
|
|
1513
1192
|
});
|
|
1514
1193
|
// --- helpers ---
|
|
1515
1194
|
withApiOptions(program.command('helpers').description('List available helper functions')).action(async (opts) => {
|
|
1516
1195
|
await handleErrors(async () => {
|
|
1517
|
-
const
|
|
1518
|
-
const
|
|
1519
|
-
const tplPath = encodeURIComponent('$.templateHelpers');
|
|
1196
|
+
const client = apiClient(opts);
|
|
1197
|
+
const query = (path) => client.get(`${getEndpoint('config').path}?path=${encodeURIComponent(path)}`);
|
|
1520
1198
|
const [mapResult, tplResult] = await Promise.all([
|
|
1521
|
-
|
|
1522
|
-
|
|
1199
|
+
query('$.mapHelpers'),
|
|
1200
|
+
query('$.templateHelpers'),
|
|
1523
1201
|
]);
|
|
1524
1202
|
const mapData = mapResult.result[0];
|
|
1525
1203
|
const tplData = tplResult.result[0];
|
|
@@ -1539,58 +1217,6 @@ function registerCustomCommands(program) {
|
|
|
1539
1217
|
})();
|
|
1540
1218
|
});
|
|
1541
1219
|
}
|
|
1542
|
-
/** Parse --json and --key options into a metadata object. Returns null on validation failure. */
|
|
1543
|
-
function parseMetadataArgs(json, keys) {
|
|
1544
|
-
let metadata = {};
|
|
1545
|
-
if (json) {
|
|
1546
|
-
try {
|
|
1547
|
-
metadata = JSON.parse(json);
|
|
1548
|
-
}
|
|
1549
|
-
catch {
|
|
1550
|
-
console.error('Invalid JSON:', json);
|
|
1551
|
-
process.exitCode = 1;
|
|
1552
|
-
return null;
|
|
1553
|
-
}
|
|
1554
|
-
}
|
|
1555
|
-
if (Array.isArray(keys) && keys.length > 0) {
|
|
1556
|
-
for (const pair of keys) {
|
|
1557
|
-
const eqIndex = pair.indexOf('=');
|
|
1558
|
-
if (eqIndex === -1) {
|
|
1559
|
-
console.error(`Invalid key-value pair: ${pair}`);
|
|
1560
|
-
process.exitCode = 1;
|
|
1561
|
-
return null;
|
|
1562
|
-
}
|
|
1563
|
-
metadata[pair.slice(0, eqIndex)] = pair.slice(eqIndex + 1);
|
|
1564
|
-
}
|
|
1565
|
-
}
|
|
1566
|
-
if (Object.keys(metadata).length === 0) {
|
|
1567
|
-
console.error('No metadata provided. Use --key or --json.');
|
|
1568
|
-
process.exitCode = 1;
|
|
1569
|
-
return null;
|
|
1570
|
-
}
|
|
1571
|
-
return metadata;
|
|
1572
|
-
}
|
|
1573
|
-
/** Format a helpers section for display. */
|
|
1574
|
-
function formatHelperSection(title, data) {
|
|
1575
|
-
if (!data || Object.keys(data).length === 0)
|
|
1576
|
-
return '';
|
|
1577
|
-
const lines = [`${title}:`];
|
|
1578
|
-
for (const [namespace, entry] of Object.entries(data)) {
|
|
1579
|
-
if (entry.description) {
|
|
1580
|
-
lines.push(` [${namespace}] ${entry.description}`);
|
|
1581
|
-
}
|
|
1582
|
-
if (entry.exports) {
|
|
1583
|
-
for (const [name, desc] of Object.entries(entry.exports)) {
|
|
1584
|
-
const descPart = desc ? ` - ${desc}` : '';
|
|
1585
|
-
lines.push(` ${name.padEnd(40)}${descPart}`);
|
|
1586
|
-
}
|
|
1587
|
-
}
|
|
1588
|
-
else {
|
|
1589
|
-
lines.push(` (no exports introspected)`);
|
|
1590
|
-
}
|
|
1591
|
-
}
|
|
1592
|
-
return lines.join('\n');
|
|
1593
|
-
}
|
|
1594
1220
|
|
|
1595
1221
|
/**
|
|
1596
1222
|
* @module descriptor
|
|
@@ -1657,4 +1283,4 @@ const watcherDescriptor = {
|
|
|
1657
1283
|
const program = createServiceCli(watcherDescriptor);
|
|
1658
1284
|
program.parse();
|
|
1659
1285
|
|
|
1660
|
-
export {
|
|
1286
|
+
export { validateFacetTypes as a, mergeInferenceRules as b, findReservedFieldWarnings as c, startFromConfig as d, findUnknownSourceKeys as f, loadConfig as l, mergeSchemas as m, resolveAndCoerce as r, shutdownComponents as s, validateSchemaCompleteness as v };
|