@wirecat/cli-core 0.344.0 → 0.346.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +64 -0
- package/dist/cli/bot/command.js +3 -1
- package/dist/cli/command-contract.d.ts +13 -0
- package/dist/cli/command-contract.js +1 -1
- package/dist/cli/context.d.ts +2 -0
- package/dist/cli/guide.d.ts +52 -0
- package/dist/cli/guide.js +307 -0
- package/dist/cli/help/command-help.d.ts +30 -0
- package/dist/cli/help/command-help.js +218 -0
- package/dist/cli/help/commands-text.d.ts +2 -0
- package/dist/cli/help/commands-text.js +1 -1
- package/dist/cli/help/first-screen.d.ts +4 -3
- package/dist/cli/help/first-screen.js +6 -3
- package/dist/cli/help/sections.js +1 -0
- package/dist/cli/index.d.ts +2 -0
- package/dist/cli/index.js +2 -0
- package/dist/cli/messenger/chat-answers.js +1 -1
- package/dist/cli/messenger/conversations-command.js +4 -1
- package/dist/cli/messenger/inbox.js +1 -1
- package/dist/cli/messenger/observed.js +15 -9
- package/dist/cli/messenger/private-people-command.js +10 -1
- package/dist/cli/messenger/review.js +1 -1
- package/dist/cli/messenger/store-maintenance-command.d.ts +1 -1
- package/dist/cli/messenger/store-maintenance-command.js +8 -5
- package/dist/cli/profile.js +1 -1
- package/dist/cli/program.js +2 -2
- package/dist/domain/models.d.ts +6 -1
- package/dist/kit/contract.d.ts +2 -2
- package/dist/kit/contract.js +36 -0
- package/dist/kit/fake.d.ts +3 -2
- package/dist/kit/fake.js +12 -0
- package/dist/meetings/mcp/index.d.ts +15 -15
- package/dist/services/index.d.ts +3 -0
- package/dist/services/index.js +1 -0
- package/dist/services/project-gathering-request.d.ts +120 -0
- package/dist/services/project-gathering-request.js +217 -0
- package/dist/services/project-gathering.d.ts +89 -0
- package/dist/services/project-gathering.js +202 -0
- package/dist/store/sqlite/chats.js +25 -4
- package/package.json +5 -4
- package/parity.json +2 -2
package/dist/kit/fake.d.ts
CHANGED
|
@@ -1,7 +1,8 @@
|
|
|
1
|
-
import type { AccountEditing, AccountTools, ChatFolders, ChatReading, ContactBook, GroupAdmin, GroupModeration, LiveUpdates, MessageEditing, MessageMedia, MessagePins, MessagePolls, MessageReactions, MessageStars, MessengerCore, PushedHistory, ReadState, ScheduledMessages, ServerReads, TopicReadState } from "../ports/messenger.js";
|
|
1
|
+
import type { AccountEditing, AccountTools, ChatFolders, ChatReading, ContactBook, GroupAdmin, GroupModeration, LiveUpdates, MessageEditing, MessageMedia, MessagePins, MessagePolls, MessageReactions, MessageStars, MessengerCore, PassiveUpdates, PushedHistory, ReadState, ScheduledMessages, ServerReads, TopicReadState } from "../ports/messenger.js";
|
|
2
2
|
import { type Seed } from "./seed.js";
|
|
3
3
|
/** Every group of the port, over memory. */
|
|
4
|
-
export type FakeAdapter = MessengerCore & ServerReads & ChatReading & MessageEditing & MessagePins & MessageStars & MessageReactions & ReadState & TopicReadState & MessagePolls & LiveUpdates & MessageMedia & ScheduledMessages & GroupModeration & AccountTools & GroupAdmin & ChatFolders & ContactBook & AccountEditing & Partial<PushedHistory>;
|
|
4
|
+
export type FakeAdapter = MessengerCore & ServerReads & ChatReading & MessageEditing & MessagePins & MessageStars & MessageReactions & ReadState & TopicReadState & MessagePolls & LiveUpdates & MessageMedia & ScheduledMessages & GroupModeration & AccountTools & GroupAdmin & ChatFolders & ContactBook & AccountEditing & Partial<PushedHistory> & Partial<PassiveUpdates>;
|
|
5
5
|
export declare const fakeAdapter: (seed?: Seed, options?: {
|
|
6
6
|
feed?: boolean;
|
|
7
|
+
listen?: boolean;
|
|
7
8
|
}) => FakeAdapter;
|
package/dist/kit/fake.js
CHANGED
|
@@ -28,6 +28,7 @@ const missing = (what) => new CliError("not_found", `the fake adapter has no ${w
|
|
|
28
28
|
*
|
|
29
29
|
* With `{ feed: true }` it also pushes its history, as a messenger with `history: "store"` does:
|
|
30
30
|
* the chats and people in one batch, then each chat's messages in a batch of their own.
|
|
31
|
+
* With `{ listen: true }` it also has `listen`, which hands on what `watch` does.
|
|
31
32
|
*/
|
|
32
33
|
const withRules = (folder, { emoji, include, skip, exclude, pin }) => {
|
|
33
34
|
if (emoji !== undefined)
|
|
@@ -311,6 +312,17 @@ export const fakeAdapter = (seed = contractSeed(), options = {}) => {
|
|
|
311
312
|
await new Promise((resolve) => signal.addEventListener("abort", () => resolve(), { once: true }));
|
|
312
313
|
listeners.delete(onEvent);
|
|
313
314
|
},
|
|
315
|
+
...(options.listen
|
|
316
|
+
? {
|
|
317
|
+
listen: async (onEvent, signal) => {
|
|
318
|
+
if (signal.aborted)
|
|
319
|
+
return;
|
|
320
|
+
listeners.add(onEvent);
|
|
321
|
+
await new Promise((resolve) => signal.addEventListener("abort", () => resolve(), { once: true }));
|
|
322
|
+
listeners.delete(onEvent);
|
|
323
|
+
},
|
|
324
|
+
}
|
|
325
|
+
: {}),
|
|
314
326
|
...(options.feed
|
|
315
327
|
? {
|
|
316
328
|
feed: async (onBatch, signal) => {
|
|
@@ -38,6 +38,7 @@ export declare const meetingTools: ({
|
|
|
38
38
|
max_read_bytes?: undefined;
|
|
39
39
|
cursor?: undefined;
|
|
40
40
|
} | {
|
|
41
|
+
limit?: undefined;
|
|
41
42
|
event_id?: undefined;
|
|
42
43
|
series_id?: undefined;
|
|
43
44
|
page?: undefined;
|
|
@@ -53,11 +54,11 @@ export declare const meetingTools: ({
|
|
|
53
54
|
format?: undefined;
|
|
54
55
|
transcript_id?: undefined;
|
|
55
56
|
query?: undefined;
|
|
56
|
-
limit?: undefined;
|
|
57
57
|
scan_limit?: undefined;
|
|
58
58
|
max_read_bytes?: undefined;
|
|
59
59
|
cursor?: undefined;
|
|
60
60
|
} | {
|
|
61
|
+
limit?: undefined;
|
|
61
62
|
event_id?: undefined;
|
|
62
63
|
series_id?: undefined;
|
|
63
64
|
page?: undefined;
|
|
@@ -75,11 +76,11 @@ export declare const meetingTools: ({
|
|
|
75
76
|
format?: undefined;
|
|
76
77
|
transcript_id?: undefined;
|
|
77
78
|
query?: undefined;
|
|
78
|
-
limit?: undefined;
|
|
79
79
|
scan_limit?: undefined;
|
|
80
80
|
max_read_bytes?: undefined;
|
|
81
81
|
cursor?: undefined;
|
|
82
82
|
} | {
|
|
83
|
+
limit?: undefined;
|
|
83
84
|
event_id?: undefined;
|
|
84
85
|
series_id?: undefined;
|
|
85
86
|
page?: undefined;
|
|
@@ -101,7 +102,6 @@ export declare const meetingTools: ({
|
|
|
101
102
|
minimum: number;
|
|
102
103
|
};
|
|
103
104
|
query?: undefined;
|
|
104
|
-
limit?: undefined;
|
|
105
105
|
scan_limit?: undefined;
|
|
106
106
|
max_read_bytes?: undefined;
|
|
107
107
|
cursor?: undefined;
|
|
@@ -188,6 +188,7 @@ export declare const meetingTools: ({
|
|
|
188
188
|
max_read_bytes?: undefined;
|
|
189
189
|
cursor?: undefined;
|
|
190
190
|
} | {
|
|
191
|
+
limit?: undefined;
|
|
191
192
|
event_id?: undefined;
|
|
192
193
|
series_id?: undefined;
|
|
193
194
|
page?: undefined;
|
|
@@ -203,11 +204,11 @@ export declare const meetingTools: ({
|
|
|
203
204
|
format?: undefined;
|
|
204
205
|
transcript_id?: undefined;
|
|
205
206
|
query?: undefined;
|
|
206
|
-
limit?: undefined;
|
|
207
207
|
scan_limit?: undefined;
|
|
208
208
|
max_read_bytes?: undefined;
|
|
209
209
|
cursor?: undefined;
|
|
210
210
|
} | {
|
|
211
|
+
limit?: undefined;
|
|
211
212
|
event_id?: undefined;
|
|
212
213
|
series_id?: undefined;
|
|
213
214
|
page?: undefined;
|
|
@@ -225,11 +226,11 @@ export declare const meetingTools: ({
|
|
|
225
226
|
format?: undefined;
|
|
226
227
|
transcript_id?: undefined;
|
|
227
228
|
query?: undefined;
|
|
228
|
-
limit?: undefined;
|
|
229
229
|
scan_limit?: undefined;
|
|
230
230
|
max_read_bytes?: undefined;
|
|
231
231
|
cursor?: undefined;
|
|
232
232
|
} | {
|
|
233
|
+
limit?: undefined;
|
|
233
234
|
event_id?: undefined;
|
|
234
235
|
series_id?: undefined;
|
|
235
236
|
page?: undefined;
|
|
@@ -251,7 +252,6 @@ export declare const meetingTools: ({
|
|
|
251
252
|
minimum: number;
|
|
252
253
|
};
|
|
253
254
|
query?: undefined;
|
|
254
|
-
limit?: undefined;
|
|
255
255
|
scan_limit?: undefined;
|
|
256
256
|
max_read_bytes?: undefined;
|
|
257
257
|
cursor?: undefined;
|
|
@@ -338,6 +338,7 @@ export declare const meetingTools: ({
|
|
|
338
338
|
max_read_bytes?: undefined;
|
|
339
339
|
cursor?: undefined;
|
|
340
340
|
} | {
|
|
341
|
+
limit?: undefined;
|
|
341
342
|
event_id?: undefined;
|
|
342
343
|
series_id?: undefined;
|
|
343
344
|
page?: undefined;
|
|
@@ -353,11 +354,11 @@ export declare const meetingTools: ({
|
|
|
353
354
|
format?: undefined;
|
|
354
355
|
transcript_id?: undefined;
|
|
355
356
|
query?: undefined;
|
|
356
|
-
limit?: undefined;
|
|
357
357
|
scan_limit?: undefined;
|
|
358
358
|
max_read_bytes?: undefined;
|
|
359
359
|
cursor?: undefined;
|
|
360
360
|
} | {
|
|
361
|
+
limit?: undefined;
|
|
361
362
|
event_id?: undefined;
|
|
362
363
|
series_id?: undefined;
|
|
363
364
|
page?: undefined;
|
|
@@ -375,11 +376,11 @@ export declare const meetingTools: ({
|
|
|
375
376
|
format?: undefined;
|
|
376
377
|
transcript_id?: undefined;
|
|
377
378
|
query?: undefined;
|
|
378
|
-
limit?: undefined;
|
|
379
379
|
scan_limit?: undefined;
|
|
380
380
|
max_read_bytes?: undefined;
|
|
381
381
|
cursor?: undefined;
|
|
382
382
|
} | {
|
|
383
|
+
limit?: undefined;
|
|
383
384
|
event_id?: undefined;
|
|
384
385
|
series_id?: undefined;
|
|
385
386
|
page?: undefined;
|
|
@@ -401,7 +402,6 @@ export declare const meetingTools: ({
|
|
|
401
402
|
minimum: number;
|
|
402
403
|
};
|
|
403
404
|
query?: undefined;
|
|
404
|
-
limit?: undefined;
|
|
405
405
|
scan_limit?: undefined;
|
|
406
406
|
max_read_bytes?: undefined;
|
|
407
407
|
cursor?: undefined;
|
|
@@ -488,6 +488,7 @@ export declare const meetingTools: ({
|
|
|
488
488
|
max_read_bytes?: undefined;
|
|
489
489
|
cursor?: undefined;
|
|
490
490
|
} | {
|
|
491
|
+
limit?: undefined;
|
|
491
492
|
event_id?: undefined;
|
|
492
493
|
series_id?: undefined;
|
|
493
494
|
page?: undefined;
|
|
@@ -503,11 +504,11 @@ export declare const meetingTools: ({
|
|
|
503
504
|
format?: undefined;
|
|
504
505
|
transcript_id?: undefined;
|
|
505
506
|
query?: undefined;
|
|
506
|
-
limit?: undefined;
|
|
507
507
|
scan_limit?: undefined;
|
|
508
508
|
max_read_bytes?: undefined;
|
|
509
509
|
cursor?: undefined;
|
|
510
510
|
} | {
|
|
511
|
+
limit?: undefined;
|
|
511
512
|
event_id?: undefined;
|
|
512
513
|
series_id?: undefined;
|
|
513
514
|
page?: undefined;
|
|
@@ -525,11 +526,11 @@ export declare const meetingTools: ({
|
|
|
525
526
|
format?: undefined;
|
|
526
527
|
transcript_id?: undefined;
|
|
527
528
|
query?: undefined;
|
|
528
|
-
limit?: undefined;
|
|
529
529
|
scan_limit?: undefined;
|
|
530
530
|
max_read_bytes?: undefined;
|
|
531
531
|
cursor?: undefined;
|
|
532
532
|
} | {
|
|
533
|
+
limit?: undefined;
|
|
533
534
|
event_id?: undefined;
|
|
534
535
|
series_id?: undefined;
|
|
535
536
|
page?: undefined;
|
|
@@ -551,7 +552,6 @@ export declare const meetingTools: ({
|
|
|
551
552
|
minimum: number;
|
|
552
553
|
};
|
|
553
554
|
query?: undefined;
|
|
554
|
-
limit?: undefined;
|
|
555
555
|
scan_limit?: undefined;
|
|
556
556
|
max_read_bytes?: undefined;
|
|
557
557
|
cursor?: undefined;
|
|
@@ -638,6 +638,7 @@ export declare const meetingTools: ({
|
|
|
638
638
|
max_read_bytes?: undefined;
|
|
639
639
|
cursor?: undefined;
|
|
640
640
|
} | {
|
|
641
|
+
limit?: undefined;
|
|
641
642
|
event_id?: undefined;
|
|
642
643
|
series_id?: undefined;
|
|
643
644
|
page?: undefined;
|
|
@@ -653,11 +654,11 @@ export declare const meetingTools: ({
|
|
|
653
654
|
format?: undefined;
|
|
654
655
|
transcript_id?: undefined;
|
|
655
656
|
query?: undefined;
|
|
656
|
-
limit?: undefined;
|
|
657
657
|
scan_limit?: undefined;
|
|
658
658
|
max_read_bytes?: undefined;
|
|
659
659
|
cursor?: undefined;
|
|
660
660
|
} | {
|
|
661
|
+
limit?: undefined;
|
|
661
662
|
event_id?: undefined;
|
|
662
663
|
series_id?: undefined;
|
|
663
664
|
page?: undefined;
|
|
@@ -675,11 +676,11 @@ export declare const meetingTools: ({
|
|
|
675
676
|
format?: undefined;
|
|
676
677
|
transcript_id?: undefined;
|
|
677
678
|
query?: undefined;
|
|
678
|
-
limit?: undefined;
|
|
679
679
|
scan_limit?: undefined;
|
|
680
680
|
max_read_bytes?: undefined;
|
|
681
681
|
cursor?: undefined;
|
|
682
682
|
} | {
|
|
683
|
+
limit?: undefined;
|
|
683
684
|
event_id?: undefined;
|
|
684
685
|
series_id?: undefined;
|
|
685
686
|
page?: undefined;
|
|
@@ -701,7 +702,6 @@ export declare const meetingTools: ({
|
|
|
701
702
|
minimum: number;
|
|
702
703
|
};
|
|
703
704
|
query?: undefined;
|
|
704
|
-
limit?: undefined;
|
|
705
705
|
scan_limit?: undefined;
|
|
706
706
|
max_read_bytes?: undefined;
|
|
707
707
|
cursor?: undefined;
|
package/dist/services/index.d.ts
CHANGED
|
@@ -2,6 +2,9 @@ import { type BotCheckService } from "../botcheck/service.js";
|
|
|
2
2
|
export type { ContextEvidenceItem, ContextEvidencePage, ContextEvidenceReader, ContextEvidenceRequest, ContextReadLimits, ContextReadUsage, ContextSource, ScopedContextEvidenceReader, } from "./context-evidence.js";
|
|
3
3
|
export { createContextEvidenceReader } from "./context-evidence.js";
|
|
4
4
|
export { type DownloadedModel, downloadImageModel, downloadTextModel, type ModelDownloadOptions, } from "./model-downloads.js";
|
|
5
|
+
export type { ProjectEvidenceItem, ProjectGathering, ProjectGatheringPorts, ProjectGatheringStop, ProjectSourceFetcher, } from "./project-gathering.js";
|
|
6
|
+
export { gatherProjectEvidence } from "./project-gathering.js";
|
|
7
|
+
export type { ProjectGatheringBudgets, ProjectGatheringRequest, ProjectLane, ProjectLaneKind, } from "./project-gathering-request.js";
|
|
5
8
|
import { type AccountService } from "./account.js";
|
|
6
9
|
import { type AdminService } from "./admin.js";
|
|
7
10
|
import { type AdminStatisticsService } from "./admin-statistics.js";
|
package/dist/services/index.js
CHANGED
|
@@ -1,6 +1,7 @@
|
|
|
1
1
|
import { botCheckService } from "../botcheck/service.js";
|
|
2
2
|
export { createContextEvidenceReader } from "./context-evidence.js";
|
|
3
3
|
export { downloadImageModel, downloadTextModel, } from "./model-downloads.js";
|
|
4
|
+
export { gatherProjectEvidence } from "./project-gathering.js";
|
|
4
5
|
import { accountService } from "./account.js";
|
|
5
6
|
import { adminService } from "./admin.js";
|
|
6
7
|
import { adminStatisticsService } from "./admin-statistics.js";
|
|
@@ -0,0 +1,120 @@
|
|
|
1
|
+
import { type ContextSource } from "../context/evidence.js";
|
|
2
|
+
export interface ProjectGatheringRequest {
|
|
3
|
+
project: {
|
|
4
|
+
name: string;
|
|
5
|
+
background?: string;
|
|
6
|
+
aliases?: string[];
|
|
7
|
+
};
|
|
8
|
+
keywords?: string[];
|
|
9
|
+
people?: {
|
|
10
|
+
name: string;
|
|
11
|
+
reference?: string;
|
|
12
|
+
}[];
|
|
13
|
+
sources: ContextSource[];
|
|
14
|
+
scope: "explicit" | "all-connected";
|
|
15
|
+
lookbackDays?: number;
|
|
16
|
+
after?: number;
|
|
17
|
+
before?: number;
|
|
18
|
+
asOf?: number;
|
|
19
|
+
fetch?: "none" | "before-read";
|
|
20
|
+
budgets?: ProjectGatheringBudgets;
|
|
21
|
+
runKey?: string;
|
|
22
|
+
checkpoint?: string;
|
|
23
|
+
signal?: AbortSignal;
|
|
24
|
+
}
|
|
25
|
+
export interface ProjectGatheringBudgets {
|
|
26
|
+
maxReads?: number;
|
|
27
|
+
maxReadBytes?: number;
|
|
28
|
+
maxPageBytes?: number;
|
|
29
|
+
maxElapsedMs?: number;
|
|
30
|
+
maxItems?: number;
|
|
31
|
+
}
|
|
32
|
+
export type ProjectLaneKind = "project" | "alias" | "keyword" | "person";
|
|
33
|
+
export interface ProjectLane {
|
|
34
|
+
id: string;
|
|
35
|
+
kinds: ProjectLaneKind[];
|
|
36
|
+
query: string;
|
|
37
|
+
people: string[];
|
|
38
|
+
}
|
|
39
|
+
export type ProjectUnitState = "pending" | "partial" | "done" | "unavailable" | "failed" | "read-limited";
|
|
40
|
+
export interface ProjectUnitProgress {
|
|
41
|
+
state: ProjectUnitState;
|
|
42
|
+
pages: number;
|
|
43
|
+
cursor: {
|
|
44
|
+
at: number;
|
|
45
|
+
id: number;
|
|
46
|
+
scope: string;
|
|
47
|
+
} | null;
|
|
48
|
+
code: string | null;
|
|
49
|
+
}
|
|
50
|
+
export type ProjectFetchState = "fetched" | "unsupported" | "failed" | "not-requested";
|
|
51
|
+
export interface ProjectGatheringCheckpoint {
|
|
52
|
+
version: 1;
|
|
53
|
+
runKey: string;
|
|
54
|
+
fingerprint: string;
|
|
55
|
+
after: number;
|
|
56
|
+
before: number;
|
|
57
|
+
asOf: number;
|
|
58
|
+
fetched: {
|
|
59
|
+
state: ProjectFetchState;
|
|
60
|
+
history: "complete" | "partial" | "unknown" | null;
|
|
61
|
+
code: string | null;
|
|
62
|
+
}[];
|
|
63
|
+
units: ProjectUnitProgress[];
|
|
64
|
+
seen: string[];
|
|
65
|
+
totals: {
|
|
66
|
+
reads: number;
|
|
67
|
+
readBytes: number;
|
|
68
|
+
items: number;
|
|
69
|
+
invocations: number;
|
|
70
|
+
};
|
|
71
|
+
}
|
|
72
|
+
export declare const CONTEXT_CHUNK_DAYS = 90;
|
|
73
|
+
export declare const projectGatheringRequest: (input: ProjectGatheringRequest, now: number) => {
|
|
74
|
+
project: {
|
|
75
|
+
name: string;
|
|
76
|
+
background: string | null;
|
|
77
|
+
aliases: string[];
|
|
78
|
+
};
|
|
79
|
+
keywords: string[];
|
|
80
|
+
people: {
|
|
81
|
+
name: string;
|
|
82
|
+
reference: string | null;
|
|
83
|
+
}[];
|
|
84
|
+
lanes: ProjectLane[];
|
|
85
|
+
sources: ContextSource[];
|
|
86
|
+
scope: "all-connected" | "explicit";
|
|
87
|
+
lookbackDays: number | null;
|
|
88
|
+
window: {
|
|
89
|
+
after: number;
|
|
90
|
+
before: number;
|
|
91
|
+
asOf: number;
|
|
92
|
+
};
|
|
93
|
+
fetch: "before-read" | "none";
|
|
94
|
+
budgets: Required<ProjectGatheringBudgets>;
|
|
95
|
+
fingerprint: string;
|
|
96
|
+
runKey: string;
|
|
97
|
+
checkpoint: string | undefined;
|
|
98
|
+
signal: AbortSignal | undefined;
|
|
99
|
+
};
|
|
100
|
+
export type ProjectGatheringPlan = ReturnType<typeof projectGatheringRequest>;
|
|
101
|
+
/** Newest chunk first, so a cut budget keeps the most recent evidence of every lane. */
|
|
102
|
+
export declare const contextChunks: (after: number, before: number) => {
|
|
103
|
+
after: number;
|
|
104
|
+
before: number;
|
|
105
|
+
}[];
|
|
106
|
+
export declare const projectUnits: (plan: ProjectGatheringPlan, window: {
|
|
107
|
+
after: number;
|
|
108
|
+
before: number;
|
|
109
|
+
}) => {
|
|
110
|
+
lane: ProjectLane;
|
|
111
|
+
source: ContextSource;
|
|
112
|
+
sourceIndex: number;
|
|
113
|
+
chunk: {
|
|
114
|
+
after: number;
|
|
115
|
+
before: number;
|
|
116
|
+
};
|
|
117
|
+
}[];
|
|
118
|
+
export declare const seenKey: (ref: string) => string;
|
|
119
|
+
export declare const encodeCheckpoint: (checkpoint: ProjectGatheringCheckpoint) => string;
|
|
120
|
+
export declare const decodeCheckpoint: (text: string, plan: ProjectGatheringPlan) => ProjectGatheringCheckpoint;
|
|
@@ -0,0 +1,217 @@
|
|
|
1
|
+
import { createHash } from "node:crypto";
|
|
2
|
+
import { CliError } from "@wirecat/cli-base";
|
|
3
|
+
import { contextSource } from "../context/evidence.js";
|
|
4
|
+
import { contextSourceKey } from "../context/evidence-response.js";
|
|
5
|
+
const DAY = 86400000;
|
|
6
|
+
const MAX_TIME = 8_640_000_000_000_000;
|
|
7
|
+
export const CONTEXT_CHUNK_DAYS = 90;
|
|
8
|
+
const LIMITS = { lanes: 16, sources: 32, units: 512, lookbackDays: 730, people: 16 };
|
|
9
|
+
const BUDGETS = {
|
|
10
|
+
maxReads: [48, 512],
|
|
11
|
+
maxReadBytes: [2097152, 16777216],
|
|
12
|
+
maxPageBytes: [65536, 262144],
|
|
13
|
+
maxElapsedMs: [60000, 600000],
|
|
14
|
+
maxItems: [100, 500],
|
|
15
|
+
};
|
|
16
|
+
const fail = (message, param) => {
|
|
17
|
+
throw new CliError("validation_error", message, param ? { param } : undefined);
|
|
18
|
+
};
|
|
19
|
+
const known = (value, allowed, prefix) => {
|
|
20
|
+
const unknown = Object.keys(value).find((key) => !allowed.includes(key));
|
|
21
|
+
if (unknown)
|
|
22
|
+
fail(`${prefix}${unknown} is not supported`, `${prefix}${unknown}`);
|
|
23
|
+
};
|
|
24
|
+
const text = (value, name, max) => typeof value === "string" && value.trim() && Buffer.byteLength(value) <= max
|
|
25
|
+
? value.trim()
|
|
26
|
+
: fail(`${name} needs 1–${max} UTF-8 bytes`, name);
|
|
27
|
+
const whole = (value, name, low, high) => Number.isSafeInteger(value) && value >= low && value <= high
|
|
28
|
+
? value
|
|
29
|
+
: fail(`${name} must be a whole number from ${low} to ${high}`, name);
|
|
30
|
+
const laneQuery = (value, name) => {
|
|
31
|
+
const query = text(value, name, 2048);
|
|
32
|
+
const words = query.match(/[\p{L}\p{N}]+/gu) ?? [];
|
|
33
|
+
if (!words.length || words.length > 16)
|
|
34
|
+
fail(`${name} needs 1–16 literal words`, name);
|
|
35
|
+
return words.join(" ");
|
|
36
|
+
};
|
|
37
|
+
const list = (value, name, max) => {
|
|
38
|
+
if (value === undefined)
|
|
39
|
+
return [];
|
|
40
|
+
if (!Array.isArray(value) || value.length > max)
|
|
41
|
+
return fail(`${name} needs at most ${max} entries`, name);
|
|
42
|
+
return value;
|
|
43
|
+
};
|
|
44
|
+
export const projectGatheringRequest = (input, now) => {
|
|
45
|
+
if (!input || typeof input !== "object")
|
|
46
|
+
fail("request needs a project and sources");
|
|
47
|
+
known(input, [
|
|
48
|
+
"project",
|
|
49
|
+
"keywords",
|
|
50
|
+
"people",
|
|
51
|
+
"sources",
|
|
52
|
+
"scope",
|
|
53
|
+
"lookbackDays",
|
|
54
|
+
"after",
|
|
55
|
+
"before",
|
|
56
|
+
"asOf",
|
|
57
|
+
"fetch",
|
|
58
|
+
"budgets",
|
|
59
|
+
"runKey",
|
|
60
|
+
"checkpoint",
|
|
61
|
+
"signal",
|
|
62
|
+
], "");
|
|
63
|
+
if (!input.project || typeof input.project !== "object")
|
|
64
|
+
fail("project needs a name", "project");
|
|
65
|
+
known(input.project, ["name", "background", "aliases"], "project.");
|
|
66
|
+
const project = {
|
|
67
|
+
name: text(input.project.name, "project.name", 512),
|
|
68
|
+
background: input.project.background === undefined ? null : text(input.project.background, "project.background", 8192),
|
|
69
|
+
aliases: list(input.project.aliases, "project.aliases", LIMITS.lanes).map((a, i) => text(a, `project.aliases.${i}`, 512)),
|
|
70
|
+
};
|
|
71
|
+
const keywords = list(input.keywords, "keywords", LIMITS.lanes).map((k, i) => laneQuery(k, `keywords.${i}`));
|
|
72
|
+
const people = list(input.people, "people", LIMITS.people).map((person, i) => {
|
|
73
|
+
if (!person || typeof person !== "object")
|
|
74
|
+
return fail(`people.${i} needs a name`, `people.${i}`);
|
|
75
|
+
known(person, ["name", "reference"], `people.${i}.`);
|
|
76
|
+
const p = person;
|
|
77
|
+
return {
|
|
78
|
+
name: laneQuery(p.name, `people.${i}.name`),
|
|
79
|
+
reference: p.reference === undefined ? null : text(p.reference, `people.${i}.reference`, 1024),
|
|
80
|
+
};
|
|
81
|
+
});
|
|
82
|
+
if (input.scope !== "explicit" && input.scope !== "all-connected")
|
|
83
|
+
fail("scope must be explicit or all-connected; omitted scope never means every account", "scope");
|
|
84
|
+
const sourceList = list(input.sources, "sources", LIMITS.sources);
|
|
85
|
+
if (!sourceList.length)
|
|
86
|
+
fail("sources needs at least one resolved source; omitted scope never means every account", "sources");
|
|
87
|
+
const sources = sourceList.map((s) => contextSource(s));
|
|
88
|
+
if (new Set(sources.map(contextSourceKey)).size !== sources.length)
|
|
89
|
+
fail("sources lists one source twice", "sources");
|
|
90
|
+
const lanes = [];
|
|
91
|
+
const lane = (kind, query, person) => {
|
|
92
|
+
const key = query.toLowerCase();
|
|
93
|
+
const found = lanes.find((l) => l.query.toLowerCase() === key) ?? {
|
|
94
|
+
id: `l${lanes.length + 1}`,
|
|
95
|
+
kinds: [],
|
|
96
|
+
query,
|
|
97
|
+
people: [],
|
|
98
|
+
};
|
|
99
|
+
if (!lanes.includes(found))
|
|
100
|
+
lanes.push(found);
|
|
101
|
+
if (!found.kinds.includes(kind))
|
|
102
|
+
found.kinds.push(kind);
|
|
103
|
+
if (person && !found.people.includes(person))
|
|
104
|
+
found.people.push(person);
|
|
105
|
+
};
|
|
106
|
+
lane("project", laneQuery(project.name, "project.name"));
|
|
107
|
+
for (const [i, a] of project.aliases.entries())
|
|
108
|
+
lane("alias", laneQuery(a, `project.aliases.${i}`));
|
|
109
|
+
for (const k of keywords)
|
|
110
|
+
lane("keyword", k);
|
|
111
|
+
for (const p of people)
|
|
112
|
+
lane("person", p.name, p.reference ?? p.name);
|
|
113
|
+
if (lanes.length > LIMITS.lanes)
|
|
114
|
+
fail(`at most ${LIMITS.lanes} distinct search lanes`, "keywords");
|
|
115
|
+
const fetch = input.fetch ?? "none";
|
|
116
|
+
if (fetch !== "none" && fetch !== "before-read")
|
|
117
|
+
fail("fetch must be none or before-read", "fetch");
|
|
118
|
+
const explicit = input.after !== undefined || input.before !== undefined;
|
|
119
|
+
if (explicit && input.lookbackDays !== undefined)
|
|
120
|
+
fail("give lookbackDays or after/before, not both", "lookbackDays");
|
|
121
|
+
if (explicit && (input.after === undefined || input.before === undefined))
|
|
122
|
+
fail("an explicit window needs both after and before", input.after === undefined ? "after" : "before");
|
|
123
|
+
const asOf = input.asOf === undefined ? null : whole(input.asOf, "asOf", 1, MAX_TIME);
|
|
124
|
+
const lookbackDays = explicit ? null : whole(input.lookbackDays ?? 60, "lookbackDays", 1, LIMITS.lookbackDays);
|
|
125
|
+
const window = explicit
|
|
126
|
+
? (() => {
|
|
127
|
+
const after = whole(input.after, "after", 0, MAX_TIME);
|
|
128
|
+
const before = whole(input.before, "before", after + 1, MAX_TIME);
|
|
129
|
+
if (before - after > LIMITS.lookbackDays * DAY)
|
|
130
|
+
fail(`window must not exceed ${LIMITS.lookbackDays} days`, "after");
|
|
131
|
+
return { after, before, asOf: asOf ?? before };
|
|
132
|
+
})()
|
|
133
|
+
: { after: Math.max(0, (asOf ?? now) - lookbackDays * DAY), before: asOf ?? now, asOf: asOf ?? now };
|
|
134
|
+
if (input.budgets !== undefined && (!input.budgets || typeof input.budgets !== "object"))
|
|
135
|
+
fail("budgets needs whole-run limits", "budgets");
|
|
136
|
+
const given = input.budgets ?? {};
|
|
137
|
+
known(given, Object.keys(BUDGETS), "budgets.");
|
|
138
|
+
const budgets = Object.fromEntries(Object.entries(BUDGETS).map(([key, [fallback, max]]) => [
|
|
139
|
+
key,
|
|
140
|
+
whole(given[key] ?? fallback, `budgets.${key}`, 1, max),
|
|
141
|
+
]));
|
|
142
|
+
if (budgets.maxPageBytes > budgets.maxReadBytes)
|
|
143
|
+
budgets.maxPageBytes = budgets.maxReadBytes;
|
|
144
|
+
const fingerprint = createHash("sha256")
|
|
145
|
+
.update(JSON.stringify([
|
|
146
|
+
"project-gathering-v1",
|
|
147
|
+
project,
|
|
148
|
+
lanes,
|
|
149
|
+
people,
|
|
150
|
+
sources.map(contextSourceKey),
|
|
151
|
+
input.scope,
|
|
152
|
+
lookbackDays,
|
|
153
|
+
explicit ? [window.after, window.before] : null,
|
|
154
|
+
asOf,
|
|
155
|
+
fetch,
|
|
156
|
+
]))
|
|
157
|
+
.digest("hex");
|
|
158
|
+
const runKey = input.runKey === undefined ? fingerprint : text(input.runKey, "runKey", 256);
|
|
159
|
+
if (input.checkpoint !== undefined && typeof input.checkpoint !== "string")
|
|
160
|
+
fail("checkpoint must be text", "checkpoint");
|
|
161
|
+
return {
|
|
162
|
+
project,
|
|
163
|
+
keywords,
|
|
164
|
+
people,
|
|
165
|
+
lanes,
|
|
166
|
+
sources,
|
|
167
|
+
scope: input.scope,
|
|
168
|
+
lookbackDays,
|
|
169
|
+
window,
|
|
170
|
+
fetch,
|
|
171
|
+
budgets,
|
|
172
|
+
fingerprint,
|
|
173
|
+
runKey,
|
|
174
|
+
checkpoint: input.checkpoint,
|
|
175
|
+
signal: input.signal,
|
|
176
|
+
};
|
|
177
|
+
};
|
|
178
|
+
/** Newest chunk first, so a cut budget keeps the most recent evidence of every lane. */
|
|
179
|
+
export const contextChunks = (after, before) => {
|
|
180
|
+
const chunks = [];
|
|
181
|
+
for (let end = before; end > after; end -= CONTEXT_CHUNK_DAYS * DAY)
|
|
182
|
+
chunks.push({ after: Math.max(after, end - CONTEXT_CHUNK_DAYS * DAY), before: end });
|
|
183
|
+
return chunks;
|
|
184
|
+
};
|
|
185
|
+
export const projectUnits = (plan, window) => {
|
|
186
|
+
const units = contextChunks(window.after, window.before).flatMap((chunk) => plan.lanes.flatMap((lane) => plan.sources.map((source, sourceIndex) => ({ lane, source, sourceIndex, chunk }))));
|
|
187
|
+
if (units.length > LIMITS.units)
|
|
188
|
+
fail(`${units.length} lane × source × 90-day reads exceed ${LIMITS.units}; narrow the lanes, sources or window`);
|
|
189
|
+
return units;
|
|
190
|
+
};
|
|
191
|
+
export const seenKey = (ref) => createHash("sha256").update(ref).digest("base64url").slice(0, 16);
|
|
192
|
+
export const encodeCheckpoint = (checkpoint) => Buffer.from(JSON.stringify(checkpoint)).toString("base64url");
|
|
193
|
+
export const decodeCheckpoint = (text, plan) => {
|
|
194
|
+
let value;
|
|
195
|
+
try {
|
|
196
|
+
if (!/^[A-Za-z0-9_-]{1,1048576}$/.test(text))
|
|
197
|
+
throw Error();
|
|
198
|
+
value = JSON.parse(Buffer.from(text, "base64url").toString("utf8"));
|
|
199
|
+
}
|
|
200
|
+
catch {
|
|
201
|
+
return fail("invalid project gathering checkpoint", "checkpoint");
|
|
202
|
+
}
|
|
203
|
+
if (value?.version !== 1 ||
|
|
204
|
+
!Array.isArray(value.units) ||
|
|
205
|
+
!Array.isArray(value.fetched) ||
|
|
206
|
+
!Array.isArray(value.seen) ||
|
|
207
|
+
!Number.isSafeInteger(value.after) ||
|
|
208
|
+
!Number.isSafeInteger(value.before) ||
|
|
209
|
+
value.after < 0 ||
|
|
210
|
+
value.before <= value.after)
|
|
211
|
+
fail("invalid project gathering checkpoint", "checkpoint");
|
|
212
|
+
if (value.runKey !== plan.runKey)
|
|
213
|
+
fail("checkpoint belongs to another run key", "checkpoint");
|
|
214
|
+
if (value.fingerprint !== plan.fingerprint)
|
|
215
|
+
fail("this run key was started with another request; a resumed run keeps its project, lanes, sources and window", "checkpoint");
|
|
216
|
+
return value;
|
|
217
|
+
};
|
|
@@ -0,0 +1,89 @@
|
|
|
1
|
+
import type { ContextEvidenceItem, ContextEvidenceReader, ContextSource } from "../context/evidence.js";
|
|
2
|
+
import { type ProjectFetchState, type ProjectGatheringRequest, type ProjectLane } from "./project-gathering-request.js";
|
|
3
|
+
/** A connector's history fetch into the store; the store reads that follow find what it saved. */
|
|
4
|
+
export interface ProjectSourceFetcher {
|
|
5
|
+
fetch(request: {
|
|
6
|
+
source: ContextSource;
|
|
7
|
+
after: number;
|
|
8
|
+
before: number;
|
|
9
|
+
signal?: AbortSignal;
|
|
10
|
+
}): Promise<{
|
|
11
|
+
status: "fetched" | "unsupported";
|
|
12
|
+
history: "complete" | "partial" | "unknown";
|
|
13
|
+
}>;
|
|
14
|
+
}
|
|
15
|
+
export interface ProjectGatheringPorts {
|
|
16
|
+
evidence: ContextEvidenceReader;
|
|
17
|
+
fetcher?: ProjectSourceFetcher;
|
|
18
|
+
now?: () => number;
|
|
19
|
+
}
|
|
20
|
+
export interface ProjectEvidenceItem extends Omit<ContextEvidenceItem, "text"> {
|
|
21
|
+
kind: ContextSource["kind"];
|
|
22
|
+
source: number;
|
|
23
|
+
text: string;
|
|
24
|
+
textTruncated: boolean;
|
|
25
|
+
lanes: string[];
|
|
26
|
+
context: "project" | "person-only";
|
|
27
|
+
}
|
|
28
|
+
export type ProjectGatheringStop = "reads" | "read-bytes" | "elapsed" | "items" | null;
|
|
29
|
+
export declare const gatherProjectEvidence: (ports: ProjectGatheringPorts, request: ProjectGatheringRequest) => Promise<{
|
|
30
|
+
schemaVersion: 1;
|
|
31
|
+
runKey: string;
|
|
32
|
+
fingerprint: string;
|
|
33
|
+
resumed: boolean;
|
|
34
|
+
project: {
|
|
35
|
+
name: string;
|
|
36
|
+
background: string | null;
|
|
37
|
+
aliases: string[];
|
|
38
|
+
};
|
|
39
|
+
window: {
|
|
40
|
+
after: number;
|
|
41
|
+
before: number;
|
|
42
|
+
asOf: number;
|
|
43
|
+
lookbackDays: number | null;
|
|
44
|
+
basis: "explicit" | "lookback";
|
|
45
|
+
timeBasis: "per-item-time-kind";
|
|
46
|
+
};
|
|
47
|
+
scope: {
|
|
48
|
+
origin: "all-connected" | "explicit";
|
|
49
|
+
sources: ContextSource[];
|
|
50
|
+
};
|
|
51
|
+
lanes: ProjectLane[];
|
|
52
|
+
fetch: {
|
|
53
|
+
state: ProjectFetchState;
|
|
54
|
+
history: "complete" | "partial" | "unknown" | null;
|
|
55
|
+
code: string | null;
|
|
56
|
+
source: number;
|
|
57
|
+
}[];
|
|
58
|
+
units: {
|
|
59
|
+
lane: string;
|
|
60
|
+
source: number;
|
|
61
|
+
after: number;
|
|
62
|
+
before: number;
|
|
63
|
+
state: import("./project-gathering-request.js").ProjectUnitState;
|
|
64
|
+
pages: number;
|
|
65
|
+
issue: string | null;
|
|
66
|
+
}[];
|
|
67
|
+
items: ProjectEvidenceItem[];
|
|
68
|
+
matchedAgain: {
|
|
69
|
+
ref: string;
|
|
70
|
+
lanes: string[];
|
|
71
|
+
}[];
|
|
72
|
+
usage: {
|
|
73
|
+
reads: number;
|
|
74
|
+
readBytes: number;
|
|
75
|
+
items: number;
|
|
76
|
+
elapsedMs: number;
|
|
77
|
+
};
|
|
78
|
+
totals: {
|
|
79
|
+
reads: number;
|
|
80
|
+
readBytes: number;
|
|
81
|
+
items: number;
|
|
82
|
+
invocations: number;
|
|
83
|
+
};
|
|
84
|
+
stopped: ProjectGatheringStop;
|
|
85
|
+
complete: boolean;
|
|
86
|
+
completeness: "unknown";
|
|
87
|
+
checkpoint: string | null;
|
|
88
|
+
}>;
|
|
89
|
+
export type ProjectGathering = Awaited<ReturnType<typeof gatherProjectEvidence>>;
|