@wirecat/cli-core 0.395.0 → 0.397.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +63 -1
- package/README.md +1 -0
- package/dist/background/daily-archive-timer.d.ts +40 -0
- package/dist/background/daily-archive-timer.js +181 -0
- package/dist/background/index.d.ts +2 -0
- package/dist/background/index.js +1 -0
- package/dist/cli/commands-command.js +6 -0
- package/dist/cli/index.d.ts +1 -1
- package/dist/cli/index.js +1 -1
- package/dist/cli/input-fields.d.ts +3 -0
- package/dist/cli/input-fields.js +27 -0
- package/dist/cli/messenger/account-command.js +1 -0
- package/dist/cli/messenger/account-sessions-command.js +3 -1
- package/dist/cli/messenger/admin-commands.js +1 -0
- package/dist/cli/messenger/admin-contacts-command.js +9 -3
- package/dist/cli/messenger/archive-commands.js +3 -1
- package/dist/cli/messenger/attachments-command.js +32 -10
- package/dist/cli/messenger/backfill-command.js +4 -0
- package/dist/cli/messenger/chats-command.js +1 -0
- package/dist/cli/messenger/chats-members-command.js +4 -0
- package/dist/cli/messenger/chats-stats-command.js +1 -0
- package/dist/cli/messenger/chats-tracking-command.js +4 -1
- package/dist/cli/messenger/complete-command.js +1 -1
- package/dist/cli/messenger/contacts-command.js +9 -3
- package/dist/cli/messenger/conversation-answers.d.ts +5 -0
- package/dist/cli/messenger/conversations-command.js +18 -1
- package/dist/cli/messenger/counters-command.js +1 -0
- package/dist/cli/messenger/download-command.js +1 -0
- package/dist/cli/messenger/flood-command.js +1 -0
- package/dist/cli/messenger/guard-commands.js +5 -4
- package/dist/cli/messenger/inbox.js +1 -0
- package/dist/cli/messenger/mcp-command.js +6 -2
- package/dist/cli/messenger/media-archive-command.d.ts +3 -0
- package/dist/cli/messenger/media-archive-command.js +72 -0
- package/dist/cli/messenger/media-archive-move-command.d.ts +3 -0
- package/dist/cli/messenger/media-archive-move-command.js +40 -0
- package/dist/cli/messenger/media-archive-timer-command.d.ts +3 -0
- package/dist/cli/messenger/media-archive-timer-command.js +70 -0
- package/dist/cli/messenger/message-answers.d.ts +5 -0
- package/dist/cli/messenger/message-writes.d.ts +5 -0
- package/dist/cli/messenger/messages-comments-command.js +1 -0
- package/dist/cli/messenger/messages-delete-command.js +1 -1
- package/dist/cli/messenger/messages-search-command.js +1 -0
- package/dist/cli/messenger/messages-stats-command.js +1 -0
- package/dist/cli/messenger/models-command.js +3 -0
- package/dist/cli/messenger/private-people-command.js +12 -10
- package/dist/cli/messenger/rankings-command.js +2 -0
- package/dist/cli/messenger/reactions-command.js +8 -2
- package/dist/cli/messenger/retention-command.js +1 -0
- package/dist/cli/messenger/review.js +1 -0
- package/dist/cli/messenger/search-answers.js +2 -0
- package/dist/cli/messenger/search-command.js +5 -1
- package/dist/cli/messenger/searches-command.d.ts +2 -0
- package/dist/cli/messenger/searches-command.js +26 -8
- package/dist/cli/messenger/server-command.js +7 -2
- package/dist/cli/messenger/store-maintenance-command.js +7 -0
- package/dist/cli/messenger/tags-command.js +6 -2
- package/dist/cli/messenger/tasks-command.js +10 -7
- package/dist/cli/messenger/transcribe-command.js +1 -0
- package/dist/cli/messenger/watch-command.js +1 -0
- package/dist/cli/program.js +5 -2
- package/dist/entities/contact-read.d.ts +119 -0
- package/dist/entities/contact-read.js +50 -0
- package/dist/entities/evidence.d.ts +40 -0
- package/dist/entities/index.d.ts +191 -2
- package/dist/entities/index.js +7 -1
- package/dist/entities/message.d.ts +9 -0
- package/dist/entities/message.js +6 -0
- package/dist/errors/reasons.d.ts +1 -0
- package/dist/errors/reasons.js +15 -0
- package/dist/meetings/mcp/index.d.ts +15 -15
- package/dist/parity/catalogue.js +1 -1
- package/dist/parity/manifest.d.ts +3 -1
- package/dist/parity/manifest.js +2 -0
- package/dist/parity/wording.js +2 -0
- package/dist/search/lucene/registry.js +2 -2
- package/dist/sends/permissions.js +7 -0
- package/dist/services/index.d.ts +5 -0
- package/dist/services/index.js +2 -0
- package/dist/services/media-archive-files.d.ts +10 -0
- package/dist/services/media-archive-files.js +110 -0
- package/dist/services/media-archive-locks.d.ts +6 -0
- package/dist/services/media-archive-locks.js +71 -0
- package/dist/services/media-archive-manifest.d.ts +29 -0
- package/dist/services/media-archive-manifest.js +116 -0
- package/dist/services/media-archive-move.d.ts +21 -0
- package/dist/services/media-archive-move.js +163 -0
- package/dist/services/media-archive.d.ts +34 -0
- package/dist/services/media-archive.js +165 -0
- package/dist/services/messages-search.js +13 -2
- package/dist/services/messages.d.ts +5 -0
- package/dist/store/search-index.d.ts +4 -1
- package/dist/store/search-index.js +10 -1
- package/dist/store/sqlite/hit-places.d.ts +12 -0
- package/dist/store/sqlite/hit-places.js +76 -0
- package/dist/store/sqlite/lucene-fields.d.ts +8 -0
- package/dist/store/sqlite/lucene-fields.js +6 -4
- package/dist/store/sqlite/lucene.js +25 -70
- package/dist/store/sqlite/manifest.js +2 -0
- package/dist/store/sqlite/migrations.generated.js +19 -0
- package/dist/store/sqlite/note-index.d.ts +11 -1
- package/dist/store/sqlite/note-index.js +18 -6
- package/dist/store/sqlite/schema.d.ts +39 -0
- package/dist/store/sqlite/schema.js +4 -0
- package/dist/store/sqlite/transcript-query.d.ts +40 -0
- package/dist/store/sqlite/transcript-query.js +108 -0
- package/dist/store/sqlite/words.d.ts +6 -0
- package/package.json +3 -3
- package/parity.json +50 -1
|
@@ -38,6 +38,7 @@ export declare const meetingTools: ({
|
|
|
38
38
|
max_read_bytes?: undefined;
|
|
39
39
|
cursor?: undefined;
|
|
40
40
|
} | {
|
|
41
|
+
limit?: undefined;
|
|
41
42
|
event_id?: undefined;
|
|
42
43
|
series_id?: undefined;
|
|
43
44
|
page?: undefined;
|
|
@@ -53,11 +54,11 @@ export declare const meetingTools: ({
|
|
|
53
54
|
format?: undefined;
|
|
54
55
|
transcript_id?: undefined;
|
|
55
56
|
query?: undefined;
|
|
56
|
-
limit?: undefined;
|
|
57
57
|
scan_limit?: undefined;
|
|
58
58
|
max_read_bytes?: undefined;
|
|
59
59
|
cursor?: undefined;
|
|
60
60
|
} | {
|
|
61
|
+
limit?: undefined;
|
|
61
62
|
event_id?: undefined;
|
|
62
63
|
series_id?: undefined;
|
|
63
64
|
page?: undefined;
|
|
@@ -75,11 +76,11 @@ export declare const meetingTools: ({
|
|
|
75
76
|
format?: undefined;
|
|
76
77
|
transcript_id?: undefined;
|
|
77
78
|
query?: undefined;
|
|
78
|
-
limit?: undefined;
|
|
79
79
|
scan_limit?: undefined;
|
|
80
80
|
max_read_bytes?: undefined;
|
|
81
81
|
cursor?: undefined;
|
|
82
82
|
} | {
|
|
83
|
+
limit?: undefined;
|
|
83
84
|
event_id?: undefined;
|
|
84
85
|
series_id?: undefined;
|
|
85
86
|
page?: undefined;
|
|
@@ -101,7 +102,6 @@ export declare const meetingTools: ({
|
|
|
101
102
|
minimum: number;
|
|
102
103
|
};
|
|
103
104
|
query?: undefined;
|
|
104
|
-
limit?: undefined;
|
|
105
105
|
scan_limit?: undefined;
|
|
106
106
|
max_read_bytes?: undefined;
|
|
107
107
|
cursor?: undefined;
|
|
@@ -188,6 +188,7 @@ export declare const meetingTools: ({
|
|
|
188
188
|
max_read_bytes?: undefined;
|
|
189
189
|
cursor?: undefined;
|
|
190
190
|
} | {
|
|
191
|
+
limit?: undefined;
|
|
191
192
|
event_id?: undefined;
|
|
192
193
|
series_id?: undefined;
|
|
193
194
|
page?: undefined;
|
|
@@ -203,11 +204,11 @@ export declare const meetingTools: ({
|
|
|
203
204
|
format?: undefined;
|
|
204
205
|
transcript_id?: undefined;
|
|
205
206
|
query?: undefined;
|
|
206
|
-
limit?: undefined;
|
|
207
207
|
scan_limit?: undefined;
|
|
208
208
|
max_read_bytes?: undefined;
|
|
209
209
|
cursor?: undefined;
|
|
210
210
|
} | {
|
|
211
|
+
limit?: undefined;
|
|
211
212
|
event_id?: undefined;
|
|
212
213
|
series_id?: undefined;
|
|
213
214
|
page?: undefined;
|
|
@@ -225,11 +226,11 @@ export declare const meetingTools: ({
|
|
|
225
226
|
format?: undefined;
|
|
226
227
|
transcript_id?: undefined;
|
|
227
228
|
query?: undefined;
|
|
228
|
-
limit?: undefined;
|
|
229
229
|
scan_limit?: undefined;
|
|
230
230
|
max_read_bytes?: undefined;
|
|
231
231
|
cursor?: undefined;
|
|
232
232
|
} | {
|
|
233
|
+
limit?: undefined;
|
|
233
234
|
event_id?: undefined;
|
|
234
235
|
series_id?: undefined;
|
|
235
236
|
page?: undefined;
|
|
@@ -251,7 +252,6 @@ export declare const meetingTools: ({
|
|
|
251
252
|
minimum: number;
|
|
252
253
|
};
|
|
253
254
|
query?: undefined;
|
|
254
|
-
limit?: undefined;
|
|
255
255
|
scan_limit?: undefined;
|
|
256
256
|
max_read_bytes?: undefined;
|
|
257
257
|
cursor?: undefined;
|
|
@@ -338,6 +338,7 @@ export declare const meetingTools: ({
|
|
|
338
338
|
max_read_bytes?: undefined;
|
|
339
339
|
cursor?: undefined;
|
|
340
340
|
} | {
|
|
341
|
+
limit?: undefined;
|
|
341
342
|
event_id?: undefined;
|
|
342
343
|
series_id?: undefined;
|
|
343
344
|
page?: undefined;
|
|
@@ -353,11 +354,11 @@ export declare const meetingTools: ({
|
|
|
353
354
|
format?: undefined;
|
|
354
355
|
transcript_id?: undefined;
|
|
355
356
|
query?: undefined;
|
|
356
|
-
limit?: undefined;
|
|
357
357
|
scan_limit?: undefined;
|
|
358
358
|
max_read_bytes?: undefined;
|
|
359
359
|
cursor?: undefined;
|
|
360
360
|
} | {
|
|
361
|
+
limit?: undefined;
|
|
361
362
|
event_id?: undefined;
|
|
362
363
|
series_id?: undefined;
|
|
363
364
|
page?: undefined;
|
|
@@ -375,11 +376,11 @@ export declare const meetingTools: ({
|
|
|
375
376
|
format?: undefined;
|
|
376
377
|
transcript_id?: undefined;
|
|
377
378
|
query?: undefined;
|
|
378
|
-
limit?: undefined;
|
|
379
379
|
scan_limit?: undefined;
|
|
380
380
|
max_read_bytes?: undefined;
|
|
381
381
|
cursor?: undefined;
|
|
382
382
|
} | {
|
|
383
|
+
limit?: undefined;
|
|
383
384
|
event_id?: undefined;
|
|
384
385
|
series_id?: undefined;
|
|
385
386
|
page?: undefined;
|
|
@@ -401,7 +402,6 @@ export declare const meetingTools: ({
|
|
|
401
402
|
minimum: number;
|
|
402
403
|
};
|
|
403
404
|
query?: undefined;
|
|
404
|
-
limit?: undefined;
|
|
405
405
|
scan_limit?: undefined;
|
|
406
406
|
max_read_bytes?: undefined;
|
|
407
407
|
cursor?: undefined;
|
|
@@ -488,6 +488,7 @@ export declare const meetingTools: ({
|
|
|
488
488
|
max_read_bytes?: undefined;
|
|
489
489
|
cursor?: undefined;
|
|
490
490
|
} | {
|
|
491
|
+
limit?: undefined;
|
|
491
492
|
event_id?: undefined;
|
|
492
493
|
series_id?: undefined;
|
|
493
494
|
page?: undefined;
|
|
@@ -503,11 +504,11 @@ export declare const meetingTools: ({
|
|
|
503
504
|
format?: undefined;
|
|
504
505
|
transcript_id?: undefined;
|
|
505
506
|
query?: undefined;
|
|
506
|
-
limit?: undefined;
|
|
507
507
|
scan_limit?: undefined;
|
|
508
508
|
max_read_bytes?: undefined;
|
|
509
509
|
cursor?: undefined;
|
|
510
510
|
} | {
|
|
511
|
+
limit?: undefined;
|
|
511
512
|
event_id?: undefined;
|
|
512
513
|
series_id?: undefined;
|
|
513
514
|
page?: undefined;
|
|
@@ -525,11 +526,11 @@ export declare const meetingTools: ({
|
|
|
525
526
|
format?: undefined;
|
|
526
527
|
transcript_id?: undefined;
|
|
527
528
|
query?: undefined;
|
|
528
|
-
limit?: undefined;
|
|
529
529
|
scan_limit?: undefined;
|
|
530
530
|
max_read_bytes?: undefined;
|
|
531
531
|
cursor?: undefined;
|
|
532
532
|
} | {
|
|
533
|
+
limit?: undefined;
|
|
533
534
|
event_id?: undefined;
|
|
534
535
|
series_id?: undefined;
|
|
535
536
|
page?: undefined;
|
|
@@ -551,7 +552,6 @@ export declare const meetingTools: ({
|
|
|
551
552
|
minimum: number;
|
|
552
553
|
};
|
|
553
554
|
query?: undefined;
|
|
554
|
-
limit?: undefined;
|
|
555
555
|
scan_limit?: undefined;
|
|
556
556
|
max_read_bytes?: undefined;
|
|
557
557
|
cursor?: undefined;
|
|
@@ -638,6 +638,7 @@ export declare const meetingTools: ({
|
|
|
638
638
|
max_read_bytes?: undefined;
|
|
639
639
|
cursor?: undefined;
|
|
640
640
|
} | {
|
|
641
|
+
limit?: undefined;
|
|
641
642
|
event_id?: undefined;
|
|
642
643
|
series_id?: undefined;
|
|
643
644
|
page?: undefined;
|
|
@@ -653,11 +654,11 @@ export declare const meetingTools: ({
|
|
|
653
654
|
format?: undefined;
|
|
654
655
|
transcript_id?: undefined;
|
|
655
656
|
query?: undefined;
|
|
656
|
-
limit?: undefined;
|
|
657
657
|
scan_limit?: undefined;
|
|
658
658
|
max_read_bytes?: undefined;
|
|
659
659
|
cursor?: undefined;
|
|
660
660
|
} | {
|
|
661
|
+
limit?: undefined;
|
|
661
662
|
event_id?: undefined;
|
|
662
663
|
series_id?: undefined;
|
|
663
664
|
page?: undefined;
|
|
@@ -675,11 +676,11 @@ export declare const meetingTools: ({
|
|
|
675
676
|
format?: undefined;
|
|
676
677
|
transcript_id?: undefined;
|
|
677
678
|
query?: undefined;
|
|
678
|
-
limit?: undefined;
|
|
679
679
|
scan_limit?: undefined;
|
|
680
680
|
max_read_bytes?: undefined;
|
|
681
681
|
cursor?: undefined;
|
|
682
682
|
} | {
|
|
683
|
+
limit?: undefined;
|
|
683
684
|
event_id?: undefined;
|
|
684
685
|
series_id?: undefined;
|
|
685
686
|
page?: undefined;
|
|
@@ -701,7 +702,6 @@ export declare const meetingTools: ({
|
|
|
701
702
|
minimum: number;
|
|
702
703
|
};
|
|
703
704
|
query?: undefined;
|
|
704
|
-
limit?: undefined;
|
|
705
705
|
scan_limit?: undefined;
|
|
706
706
|
max_read_bytes?: undefined;
|
|
707
707
|
cursor?: undefined;
|
package/dist/parity/catalogue.js
CHANGED
|
@@ -16,7 +16,7 @@ export const renderCatalogue = (manifest) => {
|
|
|
16
16
|
add(name, `every command${where(entry)}`);
|
|
17
17
|
for (const [path, row] of Object.entries(manifest.commands))
|
|
18
18
|
for (const [name, entry] of Object.entries(row.options ?? {}))
|
|
19
|
-
add(name, `\`${path}\`${where(entry)}`);
|
|
19
|
+
add(name, `\`${path}\`${where(entry)}${typeof entry !== "string" && entry.wording ? ` — ${cell(entry.wording)}` : ""}`);
|
|
20
20
|
const rows = Object.entries(manifest.options).map(([name, option]) => {
|
|
21
21
|
const meaning = option.note ? `${option.meaning}. **${option.note}**` : option.meaning;
|
|
22
22
|
return `| \`${name}\` | ${option.value ? `\`${cell(option.value)}\`` : ""} | ${cell(meaning)} | ${option.default === undefined ? "" : `\`${cell(option.default)}\``} | ${(commands.get(name) ?? []).join(", ")} |`;
|
|
@@ -11,7 +11,9 @@ export interface Presence {
|
|
|
11
11
|
planned?: Record<Cli, string>;
|
|
12
12
|
}
|
|
13
13
|
/** An option: `"all"` alone, or where it is and why not elsewhere. */
|
|
14
|
-
export type Entry = "all" | Presence
|
|
14
|
+
export type Entry = "all" | (Presence & {
|
|
15
|
+
wording?: string;
|
|
16
|
+
});
|
|
15
17
|
export interface CommandRow extends Presence {
|
|
16
18
|
/** On a planned row: it stands for every path below it, which then have no rows of their own. */
|
|
17
19
|
subtree?: boolean;
|
package/dist/parity/manifest.js
CHANGED
|
@@ -18,6 +18,8 @@ const entryProblems = (clis, where, entry) => {
|
|
|
18
18
|
const names = entry.in === "all" ? [] : Array.isArray(entry.in) ? entry.in : undefined;
|
|
19
19
|
if (names === undefined)
|
|
20
20
|
return [`${where}: "in" is neither "all" nor a list of CLIs`];
|
|
21
|
+
if ("wording" in entry && (typeof entry.wording !== "string" || entry.wording.trim() === ""))
|
|
22
|
+
problems.push(`${where}: a wording exception needs its reason`);
|
|
21
23
|
const planned = Object.entries(entry.planned ?? {});
|
|
22
24
|
for (const name of [...names, ...planned.map(([name]) => name)])
|
|
23
25
|
if (!clis.includes(name))
|
package/dist/parity/wording.js
CHANGED
|
@@ -49,6 +49,8 @@ export const wordingProblems = (manifest, programs) => {
|
|
|
49
49
|
const sentences = clis.map((cli) => said.get(cli)?.get(where));
|
|
50
50
|
if (sentences.every((sentence) => sentence === sentences[0]))
|
|
51
51
|
return [];
|
|
52
|
+
if (typeof entry !== "string" && entry.wording && sentences.every((sentence) => sentence !== undefined))
|
|
53
|
+
return [];
|
|
52
54
|
return [`${where}: ${clis.map((cli, index) => `${cli} says "${sentences[index]}"`).join(", ")}`];
|
|
53
55
|
});
|
|
54
56
|
};
|
|
@@ -294,11 +294,11 @@ export const QUERY_FIELDS = [
|
|
|
294
294
|
{
|
|
295
295
|
name: "transcript",
|
|
296
296
|
example: "transcript:invoice",
|
|
297
|
-
index: "attachment_words
|
|
297
|
+
index: "attachment_words ∪ transcript_stems; garbled spellings from transcript_words_vocab",
|
|
298
298
|
aliases: [],
|
|
299
299
|
type: "txt",
|
|
300
300
|
operators: terms,
|
|
301
|
-
normalization: "
|
|
301
|
+
normalization: "any form of the word (Snowball), widened to spellings speech recognition garbles; what a voice message said",
|
|
302
302
|
support: "A2",
|
|
303
303
|
validate: filled,
|
|
304
304
|
},
|
|
@@ -271,6 +271,7 @@ const NAMED_WRITE_KEYS = [
|
|
|
271
271
|
"stats.contacts.top.sync-first",
|
|
272
272
|
"account.sessions.list",
|
|
273
273
|
"attachments.extract",
|
|
274
|
+
"attachments.move",
|
|
274
275
|
"attachments.text.set",
|
|
275
276
|
"bot.callbacks.answer",
|
|
276
277
|
"bot.chats.action",
|
|
@@ -388,6 +389,8 @@ export const readKeysForCommand = (path) => {
|
|
|
388
389
|
const key = keyForCommand(path);
|
|
389
390
|
if (!key)
|
|
390
391
|
return [];
|
|
392
|
+
if (path[0] === "attachments" && path[1] === "download")
|
|
393
|
+
return [key, "messages", "chats"];
|
|
391
394
|
if (path[0] === "attachments" && (path[1] === "extract" || path[1] === "show"))
|
|
392
395
|
return [key, "messages"];
|
|
393
396
|
if (path[0] === "store" && path[1] === "gaps" && path[2] === "repair")
|
|
@@ -430,6 +433,10 @@ export const keyForCommand = (path) => {
|
|
|
430
433
|
// Picture models moved from `models image` (0.270.0); a model is housekeeping wherever it is listed.
|
|
431
434
|
if (top === "images" && next === "models")
|
|
432
435
|
return null;
|
|
436
|
+
if (top === "attachments" && (next === "check" || next === "jobs"))
|
|
437
|
+
return null;
|
|
438
|
+
if (top === "attachments" && next === "download")
|
|
439
|
+
return "messages";
|
|
433
440
|
if (top === "store" && next !== undefined && STORE_MAINTENANCE.has(next))
|
|
434
441
|
return null;
|
|
435
442
|
// The agent's answers write only to the local store: their own key, so read-only messages can still link (phase 4 A10).
|
package/dist/services/index.d.ts
CHANGED
|
@@ -2,6 +2,11 @@ import { type BotCheckService } from "../botcheck/service.js";
|
|
|
2
2
|
export type { ContextEvidenceItem, ContextEvidencePage, ContextEvidenceReader, ContextEvidenceRequest, ContextIndex, ContextReadLimits, ContextReadUsage, ContextRecordReader, ContextSource, ScopedContextEvidenceReader, } from "./context-evidence.js";
|
|
3
3
|
export { contextEvidenceFingerprint, createContextEvidenceReader } from "./context-evidence.js";
|
|
4
4
|
export { excerpt } from "./excerpt.js";
|
|
5
|
+
export type { MediaArchiveOptions, MediaArchivePorts } from "./media-archive.js";
|
|
6
|
+
export { archiveMedia, checkMediaArchive } from "./media-archive.js";
|
|
7
|
+
export type { MediaArchiveEntry, MediaIdentity } from "./media-archive-manifest.js";
|
|
8
|
+
export type { MoveMediaArchiveOptions } from "./media-archive-move.js";
|
|
9
|
+
export { moveMediaArchive } from "./media-archive-move.js";
|
|
5
10
|
export { type DownloadedModel, downloadImageModel, downloadTextModel, type ModelDownloadOptions, } from "./model-downloads.js";
|
|
6
11
|
export type { ProjectEvidenceContext, ProjectEvidenceItem, ProjectGathering, ProjectGatheringPorts, ProjectGatheringStop, ProjectIndexCatchUp, ProjectSourceFetcher, } from "./project-gathering.js";
|
|
7
12
|
export { gatherProjectEvidence } from "./project-gathering.js";
|
package/dist/services/index.js
CHANGED
|
@@ -1,6 +1,8 @@
|
|
|
1
1
|
import { botCheckService } from "../botcheck/service.js";
|
|
2
2
|
export { contextEvidenceFingerprint, createContextEvidenceReader } from "./context-evidence.js";
|
|
3
3
|
export { excerpt } from "./excerpt.js";
|
|
4
|
+
export { archiveMedia, checkMediaArchive } from "./media-archive.js";
|
|
5
|
+
export { moveMediaArchive } from "./media-archive-move.js";
|
|
4
6
|
export { downloadImageModel, downloadTextModel, } from "./model-downloads.js";
|
|
5
7
|
export { gatherProjectEvidence } from "./project-gathering.js";
|
|
6
8
|
export { documentPassage, storedMessageNeighbours, storedNeighbours } from "./project-gathering-around.js";
|
|
@@ -0,0 +1,10 @@
|
|
|
1
|
+
import type { RemoteFile } from "../ports/messenger.js";
|
|
2
|
+
import { type MediaArchiveEntry, type MediaIdentity } from "./media-archive-manifest.js";
|
|
3
|
+
export declare const archiveMediaFile: (root: string, identity: MediaIdentity, kind: string, file: () => Promise<RemoteFile>, beforeWrite: () => Promise<void>, signal?: AbortSignal, metadata?: {
|
|
4
|
+
messageTime?: string;
|
|
5
|
+
mime?: string;
|
|
6
|
+
expected?: {
|
|
7
|
+
size: number;
|
|
8
|
+
sha256: string;
|
|
9
|
+
};
|
|
10
|
+
}) => Promise<MediaArchiveEntry>;
|
|
@@ -0,0 +1,110 @@
|
|
|
1
|
+
import { lstat } from "node:fs/promises";
|
|
2
|
+
import { CliError } from "@wirecat/cli-base";
|
|
3
|
+
import { NativeRecordingPartialStore, NativeRecordingPublication } from "../persistence/native/index.js";
|
|
4
|
+
import { commitMediaEntry, mediaKey, missingMediaFile, parseMediaEntry, } from "./media-archive-manifest.js";
|
|
5
|
+
const validator = '"media-completed-v1"';
|
|
6
|
+
const maximumFileBytes = 64 * 1024 * 1024 * 1024;
|
|
7
|
+
export const archiveMediaFile = async (root, identity, kind, file, beforeWrite, signal, metadata = {}) => {
|
|
8
|
+
const key = mediaKey(identity);
|
|
9
|
+
const locator = {
|
|
10
|
+
provider: identity.provider,
|
|
11
|
+
account: identity.account,
|
|
12
|
+
occurrenceId: mediaKey({ ...identity, position: 0 }),
|
|
13
|
+
assetId: key,
|
|
14
|
+
};
|
|
15
|
+
const storage = new NativeRecordingPartialStore(root, locator, {
|
|
16
|
+
prefix: ".media-partial-",
|
|
17
|
+
persistent: true,
|
|
18
|
+
maxBytes: maximumFileBytes,
|
|
19
|
+
signal,
|
|
20
|
+
});
|
|
21
|
+
const publication = new NativeRecordingPublication(root, locator, {
|
|
22
|
+
prefix: ".media-publication-",
|
|
23
|
+
filename: `${key}.bin`,
|
|
24
|
+
maxBytes: maximumFileBytes,
|
|
25
|
+
metadataLimit: 16384,
|
|
26
|
+
codec: {
|
|
27
|
+
encode: JSON.stringify,
|
|
28
|
+
decode: (value) => {
|
|
29
|
+
const one = parseMediaEntry(value);
|
|
30
|
+
if (mediaKey(one) !== key || one.kind !== kind)
|
|
31
|
+
throw new CliError("validation_error", "media recovery identity mismatch");
|
|
32
|
+
if (metadata.expected && (one.size !== metadata.expected.size || one.sha256 !== metadata.expected.sha256))
|
|
33
|
+
throw new CliError("validation_error", "media recovery checksum differs from the source");
|
|
34
|
+
if (metadata.expected && (one.messageTime !== metadata.messageTime || one.mime !== metadata.mime))
|
|
35
|
+
throw new CliError("validation_error", "media recovery metadata differs from the source");
|
|
36
|
+
return one;
|
|
37
|
+
},
|
|
38
|
+
},
|
|
39
|
+
evidence: (one) => ({ size: one.size, sha256: one.sha256, validator }),
|
|
40
|
+
signal,
|
|
41
|
+
});
|
|
42
|
+
const commit = async (one) => {
|
|
43
|
+
await beforeWrite();
|
|
44
|
+
await commitMediaEntry(root, one, beforeWrite);
|
|
45
|
+
};
|
|
46
|
+
let exists = false;
|
|
47
|
+
try {
|
|
48
|
+
await lstat(publication.finalPath);
|
|
49
|
+
exists = true;
|
|
50
|
+
}
|
|
51
|
+
catch (error) {
|
|
52
|
+
if (!missingMediaFile(error))
|
|
53
|
+
throw error;
|
|
54
|
+
}
|
|
55
|
+
const intent = await publication.read();
|
|
56
|
+
if (exists) {
|
|
57
|
+
await beforeWrite();
|
|
58
|
+
return publication.recover(storage.path, () => storage.load(locator), commit);
|
|
59
|
+
}
|
|
60
|
+
if (intent) {
|
|
61
|
+
const partial = await storage.load(locator);
|
|
62
|
+
if (!partial ||
|
|
63
|
+
partial.offset !== intent.size ||
|
|
64
|
+
partial.sha256 !== intent.sha256 ||
|
|
65
|
+
partial.validator !== validator)
|
|
66
|
+
throw new CliError("validation_error", "completed media recovery evidence is missing; choose a new destination");
|
|
67
|
+
await beforeWrite();
|
|
68
|
+
await publication.publish(storage.path, intent, () => commit(intent));
|
|
69
|
+
return intent;
|
|
70
|
+
}
|
|
71
|
+
const remote = await file();
|
|
72
|
+
if (remote.size !== undefined &&
|
|
73
|
+
(!Number.isSafeInteger(remote.size) || remote.size < 0 || remote.size > maximumFileBytes))
|
|
74
|
+
throw new CliError("validation_error", "invalid media download size");
|
|
75
|
+
await beforeWrite();
|
|
76
|
+
// No range validators are exposed by messenger streams. A cut download restarts; only finish writes a checkpoint.
|
|
77
|
+
const writer = await storage.open(locator, { validator, total: remote.size ?? null });
|
|
78
|
+
let proof;
|
|
79
|
+
try {
|
|
80
|
+
let received = 0;
|
|
81
|
+
for await (const chunk of remote.bytes()) {
|
|
82
|
+
await beforeWrite();
|
|
83
|
+
await writer.write(chunk);
|
|
84
|
+
received += chunk.length;
|
|
85
|
+
}
|
|
86
|
+
if (remote.size !== undefined && received !== remote.size)
|
|
87
|
+
throw new CliError("invalid_response", "media download ended with a different size");
|
|
88
|
+
proof = await writer.finish();
|
|
89
|
+
}
|
|
90
|
+
finally {
|
|
91
|
+
await writer.close();
|
|
92
|
+
}
|
|
93
|
+
if (metadata.expected && (proof.size !== metadata.expected.size || proof.sha256 !== metadata.expected.sha256))
|
|
94
|
+
throw new CliError("validation_error", "media source changed while copying; nothing was published");
|
|
95
|
+
const mime = (metadata.mime ?? remote.mime)?.split(";", 1)[0]?.trim().toLowerCase();
|
|
96
|
+
const entry = parseMediaEntry({
|
|
97
|
+
...identity,
|
|
98
|
+
version: 1,
|
|
99
|
+
kind,
|
|
100
|
+
path: `${key}.bin`,
|
|
101
|
+
...proof,
|
|
102
|
+
...(metadata.messageTime ? { messageTime: metadata.messageTime } : {}),
|
|
103
|
+
...(mime && mime.length <= 127 && /^[a-z0-9.+-]+\/[a-z0-9.+-]+$/.test(mime) ? { mime } : {}),
|
|
104
|
+
});
|
|
105
|
+
await beforeWrite();
|
|
106
|
+
await publication.write(entry);
|
|
107
|
+
await beforeWrite();
|
|
108
|
+
await publication.publish(storage.path, entry, () => commit(entry));
|
|
109
|
+
return entry;
|
|
110
|
+
};
|
|
@@ -0,0 +1,6 @@
|
|
|
1
|
+
/** Acquire in canonical path order, then recheck the graph before trusting relocation records. */
|
|
2
|
+
export declare const withMediaArchiveLocks: <T>(directory: string, signal: AbortSignal | undefined, run: (root: string, beforeWrite: () => Promise<void>) => Promise<T>, options?: {
|
|
3
|
+
create?: boolean;
|
|
4
|
+
additional?: readonly string[];
|
|
5
|
+
createAdditional?: boolean;
|
|
6
|
+
}) => Promise<T>;
|
|
@@ -0,0 +1,71 @@
|
|
|
1
|
+
import { mkdir, realpath } from "node:fs/promises";
|
|
2
|
+
import { join, resolve } from "node:path";
|
|
3
|
+
import { CliError } from "@wirecat/cli-base";
|
|
4
|
+
import { acquireLock } from "../files/index.js";
|
|
5
|
+
import { missingMediaFile, readMediaManifests } from "./media-archive-manifest.js";
|
|
6
|
+
const archiveLocations = async (initial, signal) => {
|
|
7
|
+
const roots = new Set();
|
|
8
|
+
const queued = new Set(initial.map((directory) => resolve(directory)));
|
|
9
|
+
const pending = [...queued];
|
|
10
|
+
for (let at = 0; at < pending.length; at++) {
|
|
11
|
+
signal?.throwIfAborted();
|
|
12
|
+
let root;
|
|
13
|
+
try {
|
|
14
|
+
root = await realpath(pending[at]);
|
|
15
|
+
}
|
|
16
|
+
catch (error) {
|
|
17
|
+
if (missingMediaFile(error))
|
|
18
|
+
continue;
|
|
19
|
+
throw error;
|
|
20
|
+
}
|
|
21
|
+
if (roots.has(root))
|
|
22
|
+
continue;
|
|
23
|
+
if (roots.size >= 64)
|
|
24
|
+
throw new CliError("validation_error", "too many linked media archive directories");
|
|
25
|
+
roots.add(root);
|
|
26
|
+
for (const entry of await readMediaManifests(root, signal))
|
|
27
|
+
if (entry.relocatedTo) {
|
|
28
|
+
const location = resolve(entry.relocatedTo);
|
|
29
|
+
if (!queued.has(location)) {
|
|
30
|
+
queued.add(location);
|
|
31
|
+
pending.push(location);
|
|
32
|
+
}
|
|
33
|
+
}
|
|
34
|
+
}
|
|
35
|
+
return [...roots].sort();
|
|
36
|
+
};
|
|
37
|
+
/** Acquire in canonical path order, then recheck the graph before trusting relocation records. */
|
|
38
|
+
export const withMediaArchiveLocks = async (directory, signal, run, options = {}) => {
|
|
39
|
+
signal?.throwIfAborted();
|
|
40
|
+
if (options.create)
|
|
41
|
+
await mkdir(directory, { recursive: true, mode: 0o700 });
|
|
42
|
+
const root = await realpath(directory);
|
|
43
|
+
if (options.createAdditional)
|
|
44
|
+
for (const other of options.additional ?? [])
|
|
45
|
+
await mkdir(other, { recursive: true, mode: 0o700 });
|
|
46
|
+
const initial = [root, ...(options.additional ?? [])];
|
|
47
|
+
for (let attempt = 0; attempt < 3; attempt++) {
|
|
48
|
+
const locations = await archiveLocations(initial, signal);
|
|
49
|
+
const leases = [];
|
|
50
|
+
try {
|
|
51
|
+
for (const location of locations)
|
|
52
|
+
leases.push(await acquireLock(join(location, ".media-writers"), { signal }));
|
|
53
|
+
const current = await archiveLocations(initial, signal);
|
|
54
|
+
if (current.some((location) => !locations.includes(location)))
|
|
55
|
+
continue;
|
|
56
|
+
const beforeWrite = async () => {
|
|
57
|
+
signal?.throwIfAborted();
|
|
58
|
+
for (const lease of leases)
|
|
59
|
+
if (!(await lease.held()))
|
|
60
|
+
throw new CliError("validation_error", "media archive writer lost its lock");
|
|
61
|
+
};
|
|
62
|
+
await beforeWrite();
|
|
63
|
+
return await run(root, beforeWrite);
|
|
64
|
+
}
|
|
65
|
+
finally {
|
|
66
|
+
for (const lease of leases.reverse())
|
|
67
|
+
await lease.release();
|
|
68
|
+
}
|
|
69
|
+
}
|
|
70
|
+
throw new CliError("validation_error", "archive locations changed during locking; run the command again");
|
|
71
|
+
};
|
|
@@ -0,0 +1,29 @@
|
|
|
1
|
+
export interface MediaIdentity {
|
|
2
|
+
provider: string;
|
|
3
|
+
account: string;
|
|
4
|
+
chat: string;
|
|
5
|
+
message: string;
|
|
6
|
+
position: number;
|
|
7
|
+
}
|
|
8
|
+
export interface MediaArchiveEntry extends MediaIdentity {
|
|
9
|
+
version: 1;
|
|
10
|
+
kind: string;
|
|
11
|
+
path: string;
|
|
12
|
+
size: number;
|
|
13
|
+
sha256: string;
|
|
14
|
+
messageTime?: string;
|
|
15
|
+
mime?: string;
|
|
16
|
+
relocatedTo?: string;
|
|
17
|
+
}
|
|
18
|
+
export declare const mediaKey: (one: MediaIdentity) => string;
|
|
19
|
+
export declare const mediaManifestName: (one: MediaIdentity) => string;
|
|
20
|
+
export declare const missingMediaFile: (error: unknown) => boolean;
|
|
21
|
+
export declare const parseMediaEntry: (value: unknown) => MediaArchiveEntry;
|
|
22
|
+
export declare const mediaFingerprint: (path: string, signal?: AbortSignal) => Promise<{
|
|
23
|
+
size: number;
|
|
24
|
+
sha256: string;
|
|
25
|
+
}>;
|
|
26
|
+
export declare const readMediaManifests: (root: string, signal?: AbortSignal) => Promise<MediaArchiveEntry[]>;
|
|
27
|
+
export declare const verifyMediaEntry: (root: string, one: MediaArchiveEntry, signal?: AbortSignal) => Promise<boolean>;
|
|
28
|
+
export declare const sameMediaCopy: (a: MediaArchiveEntry, b: MediaArchiveEntry) => boolean;
|
|
29
|
+
export declare const commitMediaEntry: (root: string, one: MediaArchiveEntry, beforePublish: () => Promise<void>) => Promise<void>;
|
|
@@ -0,0 +1,116 @@
|
|
|
1
|
+
import { createHash } from "node:crypto";
|
|
2
|
+
import { readdir } from "node:fs/promises";
|
|
3
|
+
import { isAbsolute, join, resolve } from "node:path";
|
|
4
|
+
import { CliError } from "@wirecat/cli-base";
|
|
5
|
+
import { openOrdinaryFile, readBoundedUtf8File, writeFileAtomic } from "../files/index.js";
|
|
6
|
+
export const mediaKey = (one) => createHash("sha256")
|
|
7
|
+
.update(JSON.stringify([one.provider, one.account, one.chat, one.message, one.position]))
|
|
8
|
+
.digest("hex");
|
|
9
|
+
export const mediaManifestName = (one) => `.media-archive-${mediaKey(one)}.json`;
|
|
10
|
+
export const missingMediaFile = (error) => typeof error === "object" && error !== null && "code" in error && error.code === "ENOENT";
|
|
11
|
+
export const parseMediaEntry = (value) => {
|
|
12
|
+
const one = value;
|
|
13
|
+
if (!one ||
|
|
14
|
+
typeof one !== "object" ||
|
|
15
|
+
Array.isArray(one) ||
|
|
16
|
+
one.version !== 1 ||
|
|
17
|
+
Object.keys(one).some((key) => ![
|
|
18
|
+
"account",
|
|
19
|
+
"chat",
|
|
20
|
+
"kind",
|
|
21
|
+
"message",
|
|
22
|
+
"path",
|
|
23
|
+
"position",
|
|
24
|
+
"provider",
|
|
25
|
+
"sha256",
|
|
26
|
+
"size",
|
|
27
|
+
"version",
|
|
28
|
+
"messageTime",
|
|
29
|
+
"mime",
|
|
30
|
+
"relocatedTo",
|
|
31
|
+
].includes(key)) ||
|
|
32
|
+
(one.messageTime !== undefined &&
|
|
33
|
+
(typeof one.messageTime !== "string" ||
|
|
34
|
+
!Number.isFinite(Date.parse(one.messageTime)) ||
|
|
35
|
+
new Date(one.messageTime).toISOString() !== one.messageTime)) ||
|
|
36
|
+
(one.mime !== undefined &&
|
|
37
|
+
(typeof one.mime !== "string" || one.mime.length > 127 || !/^[a-z0-9.+-]+\/[a-z0-9.+-]+$/.test(one.mime))) ||
|
|
38
|
+
(one.relocatedTo !== undefined &&
|
|
39
|
+
(typeof one.relocatedTo !== "string" || !isAbsolute(one.relocatedTo) || /[\p{Cc}]/u.test(one.relocatedTo))) ||
|
|
40
|
+
[one.provider, one.account, one.chat, one.message, one.kind].some((text) => typeof text !== "string" || !text.trim() || text.length > 512 || /[\0\r\n]/.test(text)) ||
|
|
41
|
+
!Number.isSafeInteger(one.position) ||
|
|
42
|
+
one.position < 0 ||
|
|
43
|
+
!Number.isSafeInteger(one.size) ||
|
|
44
|
+
one.size < 0 ||
|
|
45
|
+
typeof one.sha256 !== "string" ||
|
|
46
|
+
!/^[a-f0-9]{64}$/.test(one.sha256) ||
|
|
47
|
+
one.path !== `${mediaKey(one)}.bin`)
|
|
48
|
+
throw new CliError("validation_error", "invalid media archive manifest");
|
|
49
|
+
return { ...one };
|
|
50
|
+
};
|
|
51
|
+
export const mediaFingerprint = async (path, signal) => {
|
|
52
|
+
signal?.throwIfAborted();
|
|
53
|
+
const file = await openOrdinaryFile(path);
|
|
54
|
+
try {
|
|
55
|
+
const hash = createHash("sha256");
|
|
56
|
+
let size = 0;
|
|
57
|
+
for await (const chunk of file.createReadStream({ autoClose: false, ...(signal ? { signal } : {}) })) {
|
|
58
|
+
signal?.throwIfAborted();
|
|
59
|
+
hash.update(chunk);
|
|
60
|
+
size += chunk.length;
|
|
61
|
+
}
|
|
62
|
+
return { size, sha256: hash.digest("hex") };
|
|
63
|
+
}
|
|
64
|
+
finally {
|
|
65
|
+
await file.close();
|
|
66
|
+
}
|
|
67
|
+
};
|
|
68
|
+
export const readMediaManifests = async (root, signal) => {
|
|
69
|
+
const entries = [];
|
|
70
|
+
for (const name of await readdir(root)) {
|
|
71
|
+
signal?.throwIfAborted();
|
|
72
|
+
if (!name.startsWith(".media-archive-") || !name.endsWith(".json"))
|
|
73
|
+
continue;
|
|
74
|
+
let one;
|
|
75
|
+
try {
|
|
76
|
+
one = parseMediaEntry(JSON.parse(await readBoundedUtf8File(join(root, name), { maxBytes: 16384 })));
|
|
77
|
+
}
|
|
78
|
+
catch {
|
|
79
|
+
throw new CliError("validation_error", "invalid media archive manifest");
|
|
80
|
+
}
|
|
81
|
+
if (name !== mediaManifestName(one))
|
|
82
|
+
throw new CliError("validation_error", "media manifest identity mismatch");
|
|
83
|
+
entries.push(one);
|
|
84
|
+
}
|
|
85
|
+
return entries;
|
|
86
|
+
};
|
|
87
|
+
export const verifyMediaEntry = async (root, one, signal) => {
|
|
88
|
+
let location = resolve(root);
|
|
89
|
+
let entry = one;
|
|
90
|
+
const visited = new Set();
|
|
91
|
+
while (entry.relocatedTo !== undefined) {
|
|
92
|
+
signal?.throwIfAborted();
|
|
93
|
+
if (visited.has(location) || visited.size >= 64)
|
|
94
|
+
throw new CliError("validation_error", "media archive relocation cycle or excessive depth");
|
|
95
|
+
visited.add(location);
|
|
96
|
+
location = resolve(entry.relocatedTo);
|
|
97
|
+
const target = parseMediaEntry(JSON.parse(await readBoundedUtf8File(join(location, mediaManifestName(one)), { maxBytes: 16384 })));
|
|
98
|
+
if (!sameMediaCopy(one, target))
|
|
99
|
+
return false;
|
|
100
|
+
entry = target;
|
|
101
|
+
}
|
|
102
|
+
if (visited.has(location))
|
|
103
|
+
return false;
|
|
104
|
+
const actual = await mediaFingerprint(join(location, one.path), signal);
|
|
105
|
+
return actual.size === one.size && actual.sha256 === one.sha256;
|
|
106
|
+
};
|
|
107
|
+
export const sameMediaCopy = (a, b) => mediaKey(a) === mediaKey(b) &&
|
|
108
|
+
a.size === b.size &&
|
|
109
|
+
a.sha256 === b.sha256 &&
|
|
110
|
+
a.kind === b.kind &&
|
|
111
|
+
a.messageTime === b.messageTime &&
|
|
112
|
+
a.mime === b.mime;
|
|
113
|
+
export const commitMediaEntry = (root, one, beforePublish) => writeFileAtomic(join(root, mediaManifestName(one)), JSON.stringify(parseMediaEntry(one)), {
|
|
114
|
+
overwrite: false,
|
|
115
|
+
beforePublish,
|
|
116
|
+
});
|