advisers 0.3.1 → 0.4.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README-zh.md +13 -15
- package/README.md +32 -22
- package/THIRD-PARTY-NOTICES.md +1 -1
- package/bin/advisers.js +2769 -183
- package/index.js +1 -1
- package/lib/brief-archive.js +267 -0
- package/lib/client-pack.js +1016 -0
- package/lib/consent.js +390 -0
- package/lib/credentials.js +195 -0
- package/lib/customer-read.js +524 -0
- package/lib/error-codes.js +516 -7
- package/lib/evidence.js +928 -120
- package/lib/kb-extract.js +626 -0
- package/lib/kb-facts.js +429 -0
- package/lib/kb-pages.js +1084 -0
- package/lib/kb.js +1036 -0
- package/lib/ledger-read.js +109 -1
- package/lib/mail-errors.js +46 -0
- package/lib/mail-extract.js +351 -0
- package/lib/mail-imap.js +446 -0
- package/lib/mail-import.js +176 -0
- package/lib/mail-parse.js +223 -0
- package/lib/mail-store.js +731 -0
- package/lib/mail.js +506 -0
- package/lib/mcp-server.js +397 -11
- package/lib/notify.js +265 -7
- package/lib/policy/decision.js +315 -0
- package/lib/policy/egress.js +268 -0
- package/lib/policy/errors.js +71 -0
- package/lib/policy/index.js +99 -0
- package/lib/policy/rules.js +468 -0
- package/lib/policy/surfaces.js +141 -0
- package/lib/receipt-verify.js +283 -26
- package/lib/receivables.js +1653 -0
- package/lib/scheduler.js +453 -9
- package/lib/shell.js +1406 -0
- package/lib/source.js +98 -3
- package/lib/sqlite.js +501 -49
- package/lib/tos/index.js +2 -2
- package/lib/tos/manifest.json +6 -6
- package/lib/tos/privacy.zh-CN.md +33 -20
- package/lib/tos/safety.zh-CN.md +7 -6
- package/lib/tos/summary.zh-CN.md +3 -2
- package/lib/tos/terms.zh-CN.md +31 -16
- package/lib/tui/labels.js +1 -1
- package/lib/tui/render.js +1 -1
- package/lib/urgency.js +467 -0
- package/package.json +4 -4
package/lib/sqlite.js
CHANGED
|
@@ -35,8 +35,16 @@ import { copyFileWithRetrySync, fsyncDirSync, fsyncFileSync, rmWithRetrySync } f
|
|
|
35
35
|
* table and no stored counter: the timeline and the client views are derived
|
|
36
36
|
* from these rows by packages/core/src/customer.ts. This is NOT the wire
|
|
37
37
|
* format: LedgerSnapshot.version stays 1 (see core/src/ledger.ts).
|
|
38
|
+
*
|
|
39
|
+
* v22 (0.4.0 KB/mail/money entities): adds the knowledge-base tables
|
|
40
|
+
* (kb_documents/kb_chunks/kb_fact_links/kb_pages), the mail intake tables
|
|
41
|
+
* (mail_accounts/mail_cursors/mail_messages/mail_sync_runs) and the money
|
|
42
|
+
* sidecars (receivables/payments) in one step, plus the two KB FTS5 indexes
|
|
43
|
+
* (kb_fts over kb_chunks, kb_pages_fts over kb_pages) built by _setupKbFts()
|
|
44
|
+
* because their tokenizer is probed at runtime (the v19 rule). None of the new
|
|
45
|
+
* tables enters the wire format: LedgerSnapshot.version stays 1.
|
|
38
46
|
*/
|
|
39
|
-
export const SCHEMA_VERSION =
|
|
47
|
+
export const SCHEMA_VERSION = 22
|
|
40
48
|
/** The last schema built by the single-block legacy migration (0.2.2). */
|
|
41
49
|
const LEGACY_SCHEMA_VERSION = 19
|
|
42
50
|
const DEFAULT_BUSY_TIMEOUT_MS = 5000
|
|
@@ -444,6 +452,95 @@ const SCHEMA_MIGRATIONS_DDL = `CREATE TABLE IF NOT EXISTS schema_migrations (
|
|
|
444
452
|
);
|
|
445
453
|
`
|
|
446
454
|
|
|
455
|
+
// ---------------------------------------------------------------------------
|
|
456
|
+
// Schema v22 (0.4.0): knowledge base, mail intake and money sidecars.
|
|
457
|
+
//
|
|
458
|
+
// All four domains land in ONE migration step (even though they are independent
|
|
459
|
+
// tables) so a database is never half-migrated: the v22 step creates the KB FTS5
|
|
460
|
+
// indexes with the probed tokenizer, and every plain table below is part of
|
|
461
|
+
// SCHEMA so a fresh database gets it at creation time (CREATE TABLE IF NOT
|
|
462
|
+
// EXISTS also converges an upgraded database before the step runs).
|
|
463
|
+
//
|
|
464
|
+
// Nothing here is a fourth ledger: KB documents/chunks/pages, mail rows and
|
|
465
|
+
// receivables/payments are local-only tables, never part of the wire
|
|
466
|
+
// LedgerSnapshot (see docs/ARCHITECTURE.md). The two FTS indexes are derived
|
|
467
|
+
// and rebuildable, and mail_cursors is per-machine sync state (neither is
|
|
468
|
+
// exported; see DATA_TABLES).
|
|
469
|
+
// ---------------------------------------------------------------------------
|
|
470
|
+
const V22_DDL = `
|
|
471
|
+
-- KB (45 §3.1): L0 metadata + extracted text, no copy of the source file.
|
|
472
|
+
CREATE TABLE IF NOT EXISTS kb_documents (
|
|
473
|
+
id TEXT PRIMARY KEY, kind TEXT NOT NULL, title TEXT NOT NULL,
|
|
474
|
+
client_id TEXT, project_id TEXT, matter_id TEXT,
|
|
475
|
+
source_name TEXT, source_sha256 TEXT NOT NULL, mime TEXT,
|
|
476
|
+
size_bytes INTEGER NOT NULL, encoding TEXT, extractor TEXT NOT NULL,
|
|
477
|
+
status TEXT NOT NULL DEFAULT 'active', secrecy INTEGER NOT NULL DEFAULT 0,
|
|
478
|
+
needs_ocr INTEGER NOT NULL DEFAULT 0, truncated INTEGER NOT NULL DEFAULT 0,
|
|
479
|
+
text_bytes INTEGER NOT NULL DEFAULT 0, chunk_count INTEGER NOT NULL DEFAULT 0,
|
|
480
|
+
safety TEXT, created_at TEXT NOT NULL, captured_at TEXT NOT NULL, deleted_at TEXT
|
|
481
|
+
);
|
|
482
|
+
CREATE TABLE IF NOT EXISTS kb_chunks (
|
|
483
|
+
id TEXT NOT NULL UNIQUE, doc_id TEXT NOT NULL, seq INTEGER NOT NULL,
|
|
484
|
+
text TEXT NOT NULL, char_count INTEGER NOT NULL, PRIMARY KEY(doc_id, seq)
|
|
485
|
+
);
|
|
486
|
+
CREATE TABLE IF NOT EXISTS kb_fact_links (
|
|
487
|
+
fact_id TEXT NOT NULL, doc_id TEXT NOT NULL, chunk_id TEXT,
|
|
488
|
+
relation TEXT NOT NULL, created_at TEXT NOT NULL,
|
|
489
|
+
PRIMARY KEY(fact_id, doc_id, relation)
|
|
490
|
+
);
|
|
491
|
+
CREATE TABLE IF NOT EXISTS kb_pages (
|
|
492
|
+
id TEXT PRIMARY KEY, kind TEXT NOT NULL, subject_id TEXT NOT NULL, title TEXT NOT NULL,
|
|
493
|
+
markdown TEXT NOT NULL, json TEXT NOT NULL, source_refs TEXT NOT NULL,
|
|
494
|
+
inputs_sha256 TEXT NOT NULL, generated_at TEXT NOT NULL,
|
|
495
|
+
status TEXT NOT NULL DEFAULT 'generated', deleted_at TEXT
|
|
496
|
+
);
|
|
497
|
+
-- Mail (41 §3.6). mail_cursors is per-mailbox sync state: local only, never
|
|
498
|
+
-- exported (a lost cursor only costs one rescan; messageId dedupe is the guard).
|
|
499
|
+
CREATE TABLE IF NOT EXISTS mail_accounts (
|
|
500
|
+
id TEXT PRIMARY KEY, name TEXT NOT NULL, host TEXT NOT NULL, port INTEGER NOT NULL,
|
|
501
|
+
tls TEXT NOT NULL, username TEXT NOT NULL, mailbox TEXT NOT NULL DEFAULT 'INBOX',
|
|
502
|
+
status TEXT NOT NULL DEFAULT 'active', default_matter_id TEXT, client_id TEXT,
|
|
503
|
+
retention_days INTEGER NOT NULL DEFAULT 30, last_sync_at TEXT, last_error TEXT,
|
|
504
|
+
created_at TEXT NOT NULL, updated_at TEXT NOT NULL
|
|
505
|
+
);
|
|
506
|
+
CREATE TABLE IF NOT EXISTS mail_cursors (
|
|
507
|
+
account_id TEXT NOT NULL, mailbox TEXT NOT NULL, uidvalidity TEXT, last_uid INTEGER,
|
|
508
|
+
uidnext TEXT, status TEXT NOT NULL DEFAULT 'ok', updated_at TEXT NOT NULL,
|
|
509
|
+
PRIMARY KEY(account_id, mailbox)
|
|
510
|
+
);
|
|
511
|
+
CREATE TABLE IF NOT EXISTS mail_messages (
|
|
512
|
+
id TEXT PRIMARY KEY, account_id TEXT NOT NULL, mailbox TEXT NOT NULL,
|
|
513
|
+
uid INTEGER NOT NULL, uidvalidity TEXT, message_id TEXT, dedupe_key TEXT NOT NULL,
|
|
514
|
+
from_addr TEXT, subject TEXT, sent_at TEXT, received_at TEXT NOT NULL,
|
|
515
|
+
size_bytes INTEGER, body_text TEXT, body_sha256 TEXT,
|
|
516
|
+
attachments TEXT NOT NULL DEFAULT '[]', client_id TEXT, project_id TEXT,
|
|
517
|
+
status TEXT NOT NULL DEFAULT 'new', candidate_ids TEXT NOT NULL DEFAULT '[]',
|
|
518
|
+
purged_at TEXT, created_at TEXT NOT NULL, updated_at TEXT NOT NULL
|
|
519
|
+
);
|
|
520
|
+
CREATE TABLE IF NOT EXISTS mail_sync_runs (
|
|
521
|
+
id TEXT PRIMARY KEY, account_id TEXT NOT NULL, mailbox TEXT NOT NULL,
|
|
522
|
+
started_at TEXT NOT NULL, finished_at TEXT, status TEXT NOT NULL,
|
|
523
|
+
fetched INTEGER NOT NULL DEFAULT 0, inserted INTEGER NOT NULL DEFAULT 0,
|
|
524
|
+
skipped INTEGER NOT NULL DEFAULT 0, failed INTEGER NOT NULL DEFAULT 0,
|
|
525
|
+
uidvalidity TEXT, last_uid_before INTEGER, last_uid_after INTEGER, error_code TEXT
|
|
526
|
+
);
|
|
527
|
+
-- Money sidecars (48 §3.2): amounts are integer minor units; status is derived
|
|
528
|
+
-- from the amount + date, never stored, so it cannot fight the commitment status.
|
|
529
|
+
CREATE TABLE IF NOT EXISTS receivables (
|
|
530
|
+
id TEXT PRIMARY KEY, commitment_id TEXT NOT NULL UNIQUE, client_id TEXT NOT NULL,
|
|
531
|
+
project_id TEXT, matter_id TEXT, delivery_id TEXT,
|
|
532
|
+
amount_minor INTEGER NOT NULL CHECK(amount_minor > 0), currency TEXT NOT NULL,
|
|
533
|
+
kind TEXT NOT NULL DEFAULT 'other', memo TEXT, closed_by_payment_id TEXT,
|
|
534
|
+
written_off_at TEXT, created_at TEXT NOT NULL, updated_at TEXT NOT NULL, deleted_at TEXT
|
|
535
|
+
);
|
|
536
|
+
CREATE TABLE IF NOT EXISTS payments (
|
|
537
|
+
id TEXT PRIMARY KEY, client_id TEXT NOT NULL, commitment_id TEXT,
|
|
538
|
+
amount_minor INTEGER NOT NULL CHECK(amount_minor > 0), currency TEXT NOT NULL,
|
|
539
|
+
paid_at TEXT NOT NULL, method TEXT NOT NULL, memo TEXT,
|
|
540
|
+
created_at TEXT NOT NULL, updated_at TEXT NOT NULL, deleted_at TEXT
|
|
541
|
+
);
|
|
542
|
+
`
|
|
543
|
+
|
|
447
544
|
const SCHEMA = `
|
|
448
545
|
CREATE TABLE IF NOT EXISTS matters (
|
|
449
546
|
id TEXT PRIMARY KEY, title TEXT NOT NULL, status TEXT NOT NULL, domain TEXT NOT NULL,
|
|
@@ -575,6 +672,10 @@ CREATE TABLE IF NOT EXISTS client_notes (
|
|
|
575
672
|
kind TEXT NOT NULL, at TEXT NOT NULL, summary TEXT NOT NULL, source TEXT,
|
|
576
673
|
created_at TEXT NOT NULL, deleted_at TEXT
|
|
577
674
|
);
|
|
675
|
+
-- 0.4.0 (schema v22): KB / mail / money tables. They are defined once in
|
|
676
|
+
-- V22_DDL and interpolated here so a fresh database and an upgraded database
|
|
677
|
+
-- converge on the exact same layout.
|
|
678
|
+
${V22_DDL}
|
|
578
679
|
CREATE TABLE IF NOT EXISTS meta (key TEXT PRIMARY KEY, value TEXT NOT NULL);
|
|
579
680
|
${SCHEMA_MIGRATIONS_DDL}`
|
|
580
681
|
|
|
@@ -606,6 +707,34 @@ const FTS_TOKENIZER_ENV = 'ADVISERS_FTS_TOKENIZER'
|
|
|
606
707
|
const FTS_TRIGRAM_MIN_QUERY = 3
|
|
607
708
|
const FTS_TABLE_NAMES = ['turns_fts', 'summaries_fts']
|
|
608
709
|
const FTS_TRIGGER_NAMES = ['turns_ai', 'turns_ad', 'turns_au', 'summaries_ai', 'summaries_ad', 'summaries_au']
|
|
710
|
+
// v22 KB FTS5 indexes (0.4.0). Deliberately kept out of FTS_TABLE_NAMES: the
|
|
711
|
+
// v19 helpers (_setupFts/_dropFts/ftsDdl) own the turn indexes and their
|
|
712
|
+
// trigger names are hard-coded there, so mixing the KB indexes in would make a
|
|
713
|
+
// repair drop one set without rebuilding it. The KB pair has its own
|
|
714
|
+
// _setupKbFts/_dropKbFts helpers and its own entries in FTS_INDEX_CONFIG, and
|
|
715
|
+
// verifyFtsIntegrity() reports all four indexes in one report.
|
|
716
|
+
const KB_FTS_TABLE_NAMES = ['kb_fts', 'kb_pages_fts']
|
|
717
|
+
const KB_FTS_TRIGGER_NAMES = ['kb_chunks_ai', 'kb_chunks_ad', 'kb_chunks_au', 'kb_pages_ai', 'kb_pages_ad', 'kb_pages_au']
|
|
718
|
+
|
|
719
|
+
/**
|
|
720
|
+
* Verify/repair configuration for every FTS5 index (schema v22).
|
|
721
|
+
*
|
|
722
|
+
* verifyFtsIntegrity() iterates this table: table name + the external-content
|
|
723
|
+
* source table + the three sync triggers that must exist for the index to stay
|
|
724
|
+
* honest. Entry order is the report order (the v19 turn pair first, then the
|
|
725
|
+
* v22 KB pair); a missing table/trigger on any entry fails the whole report.
|
|
726
|
+
*/
|
|
727
|
+
const FTS_INDEX_CONFIG = Object.freeze([
|
|
728
|
+
{ table: 'turns_fts', sourceTable: 'turns', triggers: ['turns_ai', 'turns_ad', 'turns_au'] },
|
|
729
|
+
{ table: 'summaries_fts', sourceTable: 'session_summaries', triggers: ['summaries_ai', 'summaries_ad', 'summaries_au'] },
|
|
730
|
+
{ table: 'kb_fts', sourceTable: 'kb_chunks', triggers: ['kb_chunks_ai', 'kb_chunks_ad', 'kb_chunks_au'] },
|
|
731
|
+
{ table: 'kb_pages_fts', sourceTable: 'kb_pages', triggers: ['kb_pages_ai', 'kb_pages_ad', 'kb_pages_au'] },
|
|
732
|
+
])
|
|
733
|
+
|
|
734
|
+
/** Read the tokenizer captured in a virtual table's DDL (unicode61 is implicit). */
|
|
735
|
+
function ftsTokenizerFromSql(sql) {
|
|
736
|
+
return /tokenize\s*=\s*'?trigram/i.test(String(sql ?? '')) ? 'trigram' : 'unicode61'
|
|
737
|
+
}
|
|
609
738
|
|
|
610
739
|
function ftsTokenizerPreference() {
|
|
611
740
|
const raw = String(process.env[FTS_TOKENIZER_ENV] ?? '').trim().toLowerCase()
|
|
@@ -645,6 +774,39 @@ function ftsDdl(tokenizer) {
|
|
|
645
774
|
}
|
|
646
775
|
}
|
|
647
776
|
|
|
777
|
+
/** DDL for the two v22 KB FTS indexes and their sync triggers for one tokenizer. */
|
|
778
|
+
function kbFtsDdl(tokenizer) {
|
|
779
|
+
const trigram = tokenizer === 'trigram' ? ", tokenize='trigram'" : ''
|
|
780
|
+
return {
|
|
781
|
+
tables: [
|
|
782
|
+
`CREATE VIRTUAL TABLE kb_fts USING fts5(text, content='kb_chunks', content_rowid='rowid'${trigram})`,
|
|
783
|
+
`CREATE VIRTUAL TABLE kb_pages_fts USING fts5(title, markdown, content='kb_pages', content_rowid='rowid'${trigram})`,
|
|
784
|
+
],
|
|
785
|
+
triggers: [
|
|
786
|
+
`CREATE TRIGGER kb_chunks_ai AFTER INSERT ON kb_chunks BEGIN
|
|
787
|
+
INSERT INTO kb_fts(rowid, text) VALUES (new.rowid, new.text);
|
|
788
|
+
END`,
|
|
789
|
+
`CREATE TRIGGER kb_chunks_ad AFTER DELETE ON kb_chunks BEGIN
|
|
790
|
+
INSERT INTO kb_fts(kb_fts, rowid, text) VALUES ('delete', old.rowid, old.text);
|
|
791
|
+
END`,
|
|
792
|
+
`CREATE TRIGGER kb_chunks_au AFTER UPDATE ON kb_chunks BEGIN
|
|
793
|
+
INSERT INTO kb_fts(kb_fts, rowid, text) VALUES ('delete', old.rowid, old.text);
|
|
794
|
+
INSERT INTO kb_fts(rowid, text) VALUES (new.rowid, new.text);
|
|
795
|
+
END`,
|
|
796
|
+
`CREATE TRIGGER kb_pages_ai AFTER INSERT ON kb_pages BEGIN
|
|
797
|
+
INSERT INTO kb_pages_fts(rowid, title, markdown) VALUES (new.rowid, new.title, new.markdown);
|
|
798
|
+
END`,
|
|
799
|
+
`CREATE TRIGGER kb_pages_ad AFTER DELETE ON kb_pages BEGIN
|
|
800
|
+
INSERT INTO kb_pages_fts(kb_pages_fts, rowid, title, markdown) VALUES ('delete', old.rowid, old.title, old.markdown);
|
|
801
|
+
END`,
|
|
802
|
+
`CREATE TRIGGER kb_pages_au AFTER UPDATE ON kb_pages BEGIN
|
|
803
|
+
INSERT INTO kb_pages_fts(kb_pages_fts, rowid, title, markdown) VALUES ('delete', old.rowid, old.title, old.markdown);
|
|
804
|
+
INSERT INTO kb_pages_fts(rowid, title, markdown) VALUES (new.rowid, new.title, new.markdown);
|
|
805
|
+
END`,
|
|
806
|
+
],
|
|
807
|
+
}
|
|
808
|
+
}
|
|
809
|
+
|
|
648
810
|
/**
|
|
649
811
|
* P2-1: secondary indexes for the hot read/write paths (tick, today, history,
|
|
650
812
|
* deliveries). Indexes are layout only, not a schema change, so they are
|
|
@@ -680,6 +842,35 @@ CREATE INDEX IF NOT EXISTS idx_matters_project ON matters(project_id, status);
|
|
|
680
842
|
CREATE INDEX IF NOT EXISTS idx_commitments_matter ON commitments(matter_id);
|
|
681
843
|
CREATE INDEX IF NOT EXISTS idx_deliveries_matter ON deliveries(matter_id, created_at);
|
|
682
844
|
`
|
|
845
|
+
/**
|
|
846
|
+
* v22 (0.4.0 KB/mail/money entities) layout:
|
|
847
|
+
* - idx_kb_source_unique is a PARTIAL unique index: only active, non-deleted
|
|
848
|
+
* documents dedupe on the source-file sha256, so a tombstoned document can
|
|
849
|
+
* be re-added (its bytes may legitimately recur) while a racing importer
|
|
850
|
+
* still cannot create two active rows for the same input file;
|
|
851
|
+
* - idx_mail_dedupe / idx_mail_uid are UNIQUE (not partial): the sync path
|
|
852
|
+
* inserts with ON CONFLICT and relies on the database for idempotency;
|
|
853
|
+
* - the money indexes serve the per-client aging/list read;
|
|
854
|
+
* - the kb_chunks/kb_pages primary keys already cover rowid lookups.
|
|
855
|
+
* Like every secondary index they are layout only (IF NOT EXISTS on open) and
|
|
856
|
+
* do not themselves bump the schema version.
|
|
857
|
+
*/
|
|
858
|
+
const V22_INDEXES = `
|
|
859
|
+
CREATE INDEX IF NOT EXISTS idx_kb_source_active ON kb_documents(source_sha256) WHERE status='active' AND deleted_at IS NULL;
|
|
860
|
+
CREATE UNIQUE INDEX IF NOT EXISTS idx_kb_source_unique ON kb_documents(source_sha256) WHERE status='active' AND deleted_at IS NULL;
|
|
861
|
+
CREATE INDEX IF NOT EXISTS idx_kb_client_status ON kb_documents(client_id, status);
|
|
862
|
+
CREATE INDEX IF NOT EXISTS idx_kb_matter_status ON kb_documents(matter_id, status);
|
|
863
|
+
CREATE INDEX IF NOT EXISTS idx_kb_chunks_doc ON kb_chunks(doc_id);
|
|
864
|
+
CREATE INDEX IF NOT EXISTS idx_kb_fact_links_fact ON kb_fact_links(fact_id, doc_id);
|
|
865
|
+
CREATE INDEX IF NOT EXISTS idx_kb_pages_kind ON kb_pages(kind, subject_id);
|
|
866
|
+
CREATE UNIQUE INDEX IF NOT EXISTS idx_mail_dedupe ON mail_messages(account_id, dedupe_key);
|
|
867
|
+
CREATE UNIQUE INDEX IF NOT EXISTS idx_mail_uid ON mail_messages(account_id, mailbox, uidvalidity, uid);
|
|
868
|
+
CREATE INDEX IF NOT EXISTS idx_mail_client ON mail_messages(client_id, status);
|
|
869
|
+
CREATE INDEX IF NOT EXISTS idx_receivables_client ON receivables(client_id) WHERE deleted_at IS NULL;
|
|
870
|
+
CREATE INDEX IF NOT EXISTS idx_payments_client_day ON payments(client_id, paid_at) WHERE deleted_at IS NULL;
|
|
871
|
+
CREATE INDEX IF NOT EXISTS idx_payments_commitment ON payments(commitment_id) WHERE deleted_at IS NULL;
|
|
872
|
+
`
|
|
873
|
+
|
|
683
874
|
/**
|
|
684
875
|
* 0.3.0 P0-T2 (2026-11-20): two more layout-only indexes for the pet-brief read
|
|
685
876
|
* projection. Both replace a full facts scan + TEMP B-TREE on every pet-brief:
|
|
@@ -706,7 +897,8 @@ CREATE INDEX IF NOT EXISTS idx_task_runs_created ON task_runs(created_at);
|
|
|
706
897
|
CREATE INDEX IF NOT EXISTS idx_sessions_updated ON sessions(updated_at);
|
|
707
898
|
CREATE INDEX IF NOT EXISTS idx_model_calls_created ON model_calls(created_at);
|
|
708
899
|
${V20_INDEXES}
|
|
709
|
-
${V21_INDEXES}
|
|
900
|
+
${V21_INDEXES}
|
|
901
|
+
${V22_INDEXES}`
|
|
710
902
|
|
|
711
903
|
|
|
712
904
|
/**
|
|
@@ -764,6 +956,24 @@ const MIGRATIONS = [
|
|
|
764
956
|
]) addColumnIfMissing(store.db, 'deliveries', column, type)
|
|
765
957
|
},
|
|
766
958
|
},
|
|
959
|
+
{
|
|
960
|
+
version: 22,
|
|
961
|
+
name: 'kb-mail-payment-entities',
|
|
962
|
+
up(store) {
|
|
963
|
+
// Four independent domains migrate together so no database can be
|
|
964
|
+
// half-upgraded: KB, mail intake and the money sidecars have no foreign
|
|
965
|
+
// keys between them (everything is validated by the owning module), so a
|
|
966
|
+
// single idempotent step is enough. The plain tables are also part of
|
|
967
|
+
// SCHEMA and were created with IF NOT EXISTS before this step ran; the
|
|
968
|
+
// DDL is repeated here so a direct migrate() caller converges too.
|
|
969
|
+
store.db.exec(V22_DDL)
|
|
970
|
+
// The tokenizer must be probed at runtime, so the two KB FTS5 indexes are
|
|
971
|
+
// built here (not in SCHEMA), exactly like the v19 turn indexes. This is
|
|
972
|
+
// idempotent (drop + recreate + 'rebuild' from kb_chunks/kb_pages) and is
|
|
973
|
+
// also the repair path for kb reindex / audit --rebuild-fts.
|
|
974
|
+
store._setupKbFts()
|
|
975
|
+
},
|
|
976
|
+
},
|
|
767
977
|
]
|
|
768
978
|
const json = (v) => JSON.stringify(v ?? [])
|
|
769
979
|
const parse = (v, fallback = []) => {
|
|
@@ -797,15 +1007,26 @@ export const CLIENT_NOTE_SOURCE_KINDS = ['delivery', 'matter', 'manual']
|
|
|
797
1007
|
|
|
798
1008
|
/**
|
|
799
1009
|
* Tables that a full export --all carries next to the nine-table ledger
|
|
800
|
-
* snapshot. The
|
|
801
|
-
* round-trip cannot silently drop them; this
|
|
802
|
-
* CLI
|
|
803
|
-
*
|
|
1010
|
+
* snapshot. The v21 customer tables and the v22 KB/mail/money tables are
|
|
1011
|
+
* appended so an export/import round-trip cannot silently drop them; this
|
|
1012
|
+
* constant is the single list the CLI reads instead of hard-coding its own.
|
|
1013
|
+
*
|
|
1014
|
+
* Deliberately NOT in this list:
|
|
1015
|
+
* - kb_fts / kb_pages_fts (and their shadow tables): derived, rebuildable from
|
|
1016
|
+
* kb_chunks/kb_pages by _setupKbFts();
|
|
1017
|
+
* - mail_cursors: per-machine sync state, not customer data (a lost cursor
|
|
1018
|
+
* only triggers a rescan; message dedupe is the correctness guard).
|
|
1019
|
+
* kb_pages IS exported: the page markdown is a rebuildable projection, but the
|
|
1020
|
+
* export/import round-trip is still expected to restore it byte for byte so a
|
|
1021
|
+
* restored HOME shows the same pages before the first rebuild.
|
|
804
1022
|
*/
|
|
805
1023
|
export const DATA_TABLES = Object.freeze([
|
|
806
1024
|
'sessions', 'turns', 'session_summaries', 'summary_versions', 'deliveries', 'nudges',
|
|
807
1025
|
'task_runs', 'audit', 'memory_candidates', 'permit_consumed',
|
|
808
1026
|
'clients', 'projects', 'client_notes',
|
|
1027
|
+
'kb_documents', 'kb_chunks', 'kb_fact_links', 'kb_pages',
|
|
1028
|
+
'mail_accounts', 'mail_messages', 'mail_sync_runs',
|
|
1029
|
+
'receivables', 'payments',
|
|
809
1030
|
])
|
|
810
1031
|
|
|
811
1032
|
/** A business error with a stable code (message keeps the code for stderr). */
|
|
@@ -1227,12 +1448,18 @@ function encodeEntityValue(codec, value) {
|
|
|
1227
1448
|
}
|
|
1228
1449
|
|
|
1229
1450
|
/**
|
|
1230
|
-
* Tables whose row counts are sampled to verify a pre-migration backup
|
|
1231
|
-
*
|
|
1232
|
-
*
|
|
1233
|
-
*
|
|
1451
|
+
* Tables whose row counts are sampled to verify a pre-migration backup
|
|
1452
|
+
* (fail-closed: a copy that silently lost committed rows is removed and the
|
|
1453
|
+
* migration is refused). The v21 customer and v22 KB/mail/money tables are
|
|
1454
|
+
* included so a future migration's backup verification proves they were copied
|
|
1455
|
+
* too; tables absent from the old schema are skipped by _sampleTableCounts.
|
|
1234
1456
|
*/
|
|
1235
|
-
const BACKUP_SAMPLE_TABLES = [
|
|
1457
|
+
const BACKUP_SAMPLE_TABLES = [
|
|
1458
|
+
'matters', 'facts', 'commitments', 'preferences', 'audit',
|
|
1459
|
+
'clients', 'projects', 'client_notes',
|
|
1460
|
+
'kb_documents', 'kb_pages', 'mail_accounts', 'mail_messages', 'receivables', 'payments',
|
|
1461
|
+
]
|
|
1462
|
+
|
|
1236
1463
|
// ---------------------------------------------------------------------------
|
|
1237
1464
|
// WAL status / recovery (robustness audit 2026-11-17 P1-6).
|
|
1238
1465
|
//
|
|
@@ -1417,12 +1644,18 @@ export class SqliteStore {
|
|
|
1417
1644
|
this._readDepth = 0
|
|
1418
1645
|
this._baselineIds = null
|
|
1419
1646
|
this._fullLoad = false
|
|
1420
|
-
// FTS5 state (schema v19): which indexes exist and which tokenizer
|
|
1421
|
-
// them. Fillable stores discover it after migrate(); read-only stores
|
|
1422
|
-
// it from sqlite_master without running DDL (see _initFtsState).
|
|
1647
|
+
// FTS5 state (schema v19/v22): which indexes exist and which tokenizer
|
|
1648
|
+
// built them. Fillable stores discover it after migrate(); read-only stores
|
|
1649
|
+
// detect it from sqlite_master without running DDL (see _initFtsState).
|
|
1650
|
+
// ftsTokenizer/_ftsTables describe the v19 turn indexes (the legacy search
|
|
1651
|
+
// paths read them); kbFtsTokenizer/_kbFtsTables describe the v22 KB pair.
|
|
1423
1652
|
this.ftsTokenizer = null
|
|
1424
1653
|
this.ftsWarning = null
|
|
1425
1654
|
this._ftsTables = new Set()
|
|
1655
|
+
this.ftsTokenizers = {}
|
|
1656
|
+
this.kbFtsTokenizer = null
|
|
1657
|
+
this.kbFtsWarning = null
|
|
1658
|
+
this._kbFtsTables = new Set()
|
|
1426
1659
|
// P1-5: searchTurns()/searchSummaries() set these when FTS and LIKE
|
|
1427
1660
|
// disagree; verifyFtsIntegrity() fills ftsIntegrityReport for doctor.
|
|
1428
1661
|
this._ftsBroken = false
|
|
@@ -1641,13 +1874,17 @@ export class SqliteStore {
|
|
|
1641
1874
|
|
|
1642
1875
|
/** Look up which FTS indexes/tokenizer this database actually has on disk. */
|
|
1643
1876
|
_initFtsState() {
|
|
1644
|
-
const
|
|
1645
|
-
this.
|
|
1646
|
-
|
|
1647
|
-
// v19 records the tokenizer in the virtual-table DDL. An older or
|
|
1877
|
+
const names = [...FTS_TABLE_NAMES, ...KB_FTS_TABLE_NAMES]
|
|
1878
|
+
const rows = this.db.prepare(`SELECT name, sql FROM sqlite_master WHERE name IN (${names.map(() => '?').join(',')})`).all(...names)
|
|
1879
|
+
// v19/v22 record the tokenizer in the virtual-table DDL. An older or
|
|
1648
1880
|
// hand-modified database may have no index at all: search then uses the
|
|
1649
1881
|
// LIKE path only, which stays correct.
|
|
1650
|
-
this.
|
|
1882
|
+
this.ftsTokenizers = {}
|
|
1883
|
+
for (const row of rows) this.ftsTokenizers[row.name] = ftsTokenizerFromSql(row.sql)
|
|
1884
|
+
this._ftsTables = new Set(rows.filter((row) => FTS_TABLE_NAMES.includes(row.name)).map((row) => row.name))
|
|
1885
|
+
this._kbFtsTables = new Set(rows.filter((row) => KB_FTS_TABLE_NAMES.includes(row.name)).map((row) => row.name))
|
|
1886
|
+
this.ftsTokenizer = this.ftsTokenizers.turns_fts ?? null
|
|
1887
|
+
this.kbFtsTokenizer = this.ftsTokenizers.kb_fts ?? null
|
|
1651
1888
|
return this.ftsTokenizer
|
|
1652
1889
|
}
|
|
1653
1890
|
|
|
@@ -1664,6 +1901,18 @@ export class SqliteStore {
|
|
|
1664
1901
|
for (const table of FTS_TABLE_NAMES) this.db.exec(`DROP TABLE IF EXISTS ${table}`)
|
|
1665
1902
|
}
|
|
1666
1903
|
|
|
1904
|
+
_createKbFts(tokenizer) {
|
|
1905
|
+
const ddl = kbFtsDdl(tokenizer)
|
|
1906
|
+
for (const sql of ddl.tables) this.db.exec(sql)
|
|
1907
|
+
for (const sql of ddl.triggers) this.db.exec(sql)
|
|
1908
|
+
}
|
|
1909
|
+
|
|
1910
|
+
_dropKbFts() {
|
|
1911
|
+
// Same order as _dropFts: triggers first, then the virtual tables.
|
|
1912
|
+
for (const trigger of KB_FTS_TRIGGER_NAMES) this.db.exec(`DROP TRIGGER IF EXISTS ${trigger}`)
|
|
1913
|
+
for (const table of KB_FTS_TABLE_NAMES) this.db.exec(`DROP TABLE IF EXISTS ${table}`)
|
|
1914
|
+
}
|
|
1915
|
+
|
|
1667
1916
|
/**
|
|
1668
1917
|
* Install (or rebuild) the v19 FTS5 indexes with the trigram tokenizer.
|
|
1669
1918
|
*
|
|
@@ -1718,6 +1967,73 @@ export class SqliteStore {
|
|
|
1718
1967
|
return tokenizer
|
|
1719
1968
|
}
|
|
1720
1969
|
|
|
1970
|
+
/**
|
|
1971
|
+
* Install (or rebuild) the v22 KB FTS5 indexes with the probed tokenizer.
|
|
1972
|
+
*
|
|
1973
|
+
* Same contract as _setupFts(): runs inside the v22 migration transaction (or
|
|
1974
|
+
* a repair transaction), so dropping/creating both virtual tables, replacing
|
|
1975
|
+
* the six sync triggers and the 'rebuild' backfill are atomic with the schema
|
|
1976
|
+
* version bump / repair. Rows are re-indexed from kb_chunks and kb_pages, so
|
|
1977
|
+
* reindexing never needs the (unretained) source files.
|
|
1978
|
+
*
|
|
1979
|
+
* Degrades instead of crashing: trigram -> unicode61 -> LIKE-only, recording
|
|
1980
|
+
* kbFtsWarning and printing one warning line. The database still advances to
|
|
1981
|
+
* v22 so a missing tokenizer build is not retried on every open; searchKb /
|
|
1982
|
+
* kb_pages search must check kbFtsTokenizer and fall back to LIKE when null.
|
|
1983
|
+
*/
|
|
1984
|
+
_setupKbFts() {
|
|
1985
|
+
this._dropKbFts()
|
|
1986
|
+
const preference = ftsTokenizerPreference()
|
|
1987
|
+
let tokenizer = preference === 'unicode61' ? null : 'trigram'
|
|
1988
|
+
if (tokenizer) {
|
|
1989
|
+
try {
|
|
1990
|
+
this._createKbFts(tokenizer)
|
|
1991
|
+
} catch (error) {
|
|
1992
|
+
this._dropKbFts()
|
|
1993
|
+
this.kbFtsWarning = `FTS5 trigram tokenizer unavailable for the KB indexes (${error.message}); kb search falls back to LIKE`
|
|
1994
|
+
try { console.warn(`advisers: warning: ${this.kbFtsWarning}`) } catch {}
|
|
1995
|
+
tokenizer = null
|
|
1996
|
+
}
|
|
1997
|
+
} else if (preference === 'unicode61') {
|
|
1998
|
+
this.kbFtsWarning = 'ADVISERS_FTS_TOKENIZER=unicode61 forces the legacy FTS5 tokenizer; kb search falls back to LIKE for Chinese substrings'
|
|
1999
|
+
try { console.warn(`advisers: warning: ${this.kbFtsWarning}`) } catch {}
|
|
2000
|
+
}
|
|
2001
|
+
if (!tokenizer) {
|
|
2002
|
+
try {
|
|
2003
|
+
this._createKbFts('unicode61')
|
|
2004
|
+
tokenizer = 'unicode61'
|
|
2005
|
+
} catch (error) {
|
|
2006
|
+
this._dropKbFts()
|
|
2007
|
+
tokenizer = null
|
|
2008
|
+
this.kbFtsWarning = `FTS5 is unavailable in this SQLite build (${error.message}); kb search uses LIKE only`
|
|
2009
|
+
try { console.warn(`advisers: warning: ${this.kbFtsWarning}`) } catch {}
|
|
2010
|
+
}
|
|
2011
|
+
}
|
|
2012
|
+
this.kbFtsTokenizer = tokenizer
|
|
2013
|
+
this._kbFtsTables = new Set(tokenizer ? KB_FTS_TABLE_NAMES : [])
|
|
2014
|
+
for (const table of KB_FTS_TABLE_NAMES) {
|
|
2015
|
+
if (tokenizer) this.ftsTokenizers[table] = tokenizer
|
|
2016
|
+
else delete this.ftsTokenizers[table]
|
|
2017
|
+
}
|
|
2018
|
+
if (tokenizer) {
|
|
2019
|
+
this.db.exec("INSERT INTO kb_fts(kb_fts) VALUES('rebuild')")
|
|
2020
|
+
this.db.exec("INSERT INTO kb_pages_fts(kb_pages_fts) VALUES('rebuild')")
|
|
2021
|
+
}
|
|
2022
|
+
return tokenizer
|
|
2023
|
+
}
|
|
2024
|
+
|
|
2025
|
+
/**
|
|
2026
|
+
* Rebuild every FTS5 index (the v19 turn pair + the v22 KB pair) from its
|
|
2027
|
+
* source tables. This is the single repair entry point for 'advisers audit
|
|
2028
|
+
* --rebuild-fts' and 'advisers doctor --repair --only fts'; callers run it
|
|
2029
|
+
* inside one write transaction so a failure keeps the previous indexes.
|
|
2030
|
+
* Returns the installed tokenizers (null = the LIKE-only degradation).
|
|
2031
|
+
*/
|
|
2032
|
+
rebuildAllFtsIndexes() {
|
|
2033
|
+
this._assertWritable('rebuildAllFtsIndexes')
|
|
2034
|
+
return { turns: this._setupFts(), kb: this._setupKbFts() }
|
|
2035
|
+
}
|
|
2036
|
+
|
|
1721
2037
|
/**
|
|
1722
2038
|
* Report the WAL/shm state and whether this connection can checkpoint.
|
|
1723
2039
|
* `initial` is the pre-open inspection (see inspectWalStatus above); a fresh
|
|
@@ -1748,16 +2064,20 @@ export class SqliteStore {
|
|
|
1748
2064
|
}
|
|
1749
2065
|
|
|
1750
2066
|
/**
|
|
1751
|
-
* Verify
|
|
1752
|
-
* 2026-11-17 P1-5
|
|
1753
|
-
* fewer rows than a LIKE scan; this is the
|
|
1754
|
-
* audit --check-db
|
|
2067
|
+
* Verify every FTS5 index against its source table (robustness audit
|
|
2068
|
+
* 2026-11-17 P1-5; extended by v22 to the two KB indexes). A desynchronised
|
|
2069
|
+
* index makes search silently return fewer rows than a LIKE scan; this is the
|
|
2070
|
+
* explicit check that doctor / audit --check-db surface (O(index), never on
|
|
2071
|
+
* every open).
|
|
1755
2072
|
*
|
|
1756
|
-
* Checks per
|
|
1757
|
-
* 'integrity-check' (catches malformed/desynced segments),
|
|
1758
|
-
* (shadow docsize) row counts, and a deterministic rowid
|
|
1759
|
-
* directions. On a non-ok result the store also marks its FTS
|
|
1760
|
-
* (`searchTurns`/`searchSummaries` then use LIKE until a
|
|
2073
|
+
* Checks per FTS_INDEX_CONFIG entry: table present, the three sync triggers
|
|
2074
|
+
* present, FTS5 'integrity-check' (catches malformed/desynced segments),
|
|
2075
|
+
* source vs index (shadow docsize) row counts, and a deterministic rowid
|
|
2076
|
+
* sample in both directions. On a non-ok result the store also marks its FTS
|
|
2077
|
+
* path unusable (`searchTurns`/`searchSummaries` then use LIKE until a
|
|
2078
|
+
* rebuild). The legacy report.triggers and report.tokenizer fields stay
|
|
2079
|
+
* turn-index shaped for compatibility; the KB indexes appear in the additive
|
|
2080
|
+
* tokenizers/kbTokenizer fields and as their own report.tables entries.
|
|
1761
2081
|
*/
|
|
1762
2082
|
verifyFtsIntegrity({ sampleSize = 100 } = {}) {
|
|
1763
2083
|
this._initFtsState()
|
|
@@ -1765,6 +2085,8 @@ export class SqliteStore {
|
|
|
1765
2085
|
ok: true,
|
|
1766
2086
|
state: 'ok',
|
|
1767
2087
|
tokenizer: this.ftsTokenizer,
|
|
2088
|
+
kbTokenizer: this.kbFtsTokenizer,
|
|
2089
|
+
tokenizers: { ...this.ftsTokenizers },
|
|
1768
2090
|
tables: {},
|
|
1769
2091
|
triggers: { expected: FTS_TRIGGER_NAMES.slice(), missing: [] },
|
|
1770
2092
|
checkedAt: new Date().toISOString(),
|
|
@@ -1778,24 +2100,18 @@ export class SqliteStore {
|
|
|
1778
2100
|
} catch (error) {
|
|
1779
2101
|
report.errors.push(`cannot list triggers: ${String(error?.message ?? error).slice(0, 200)}`)
|
|
1780
2102
|
}
|
|
1781
|
-
const triggerMap = {
|
|
1782
|
-
turns_fts: ['turns_ai', 'turns_ad', 'turns_au'],
|
|
1783
|
-
summaries_fts: ['summaries_ai', 'summaries_ad', 'summaries_au'],
|
|
1784
|
-
}
|
|
1785
|
-
const sourceMap = {
|
|
1786
|
-
turns_fts: { table: 'turns' },
|
|
1787
|
-
summaries_fts: { table: 'session_summaries' },
|
|
1788
|
-
}
|
|
1789
2103
|
let anyTable = false
|
|
1790
2104
|
let inconsistent = false
|
|
1791
2105
|
let missingTable = false
|
|
1792
|
-
for (const
|
|
1793
|
-
const
|
|
2106
|
+
for (const config of FTS_INDEX_CONFIG) {
|
|
2107
|
+
const table = config.table
|
|
2108
|
+
const sourceTable = config.sourceTable
|
|
1794
2109
|
const info = {
|
|
1795
2110
|
table,
|
|
1796
|
-
sourceTable
|
|
2111
|
+
sourceTable,
|
|
1797
2112
|
exists: false,
|
|
1798
|
-
|
|
2113
|
+
tokenizer: this.ftsTokenizers[table] ?? null,
|
|
2114
|
+
triggers: { expected: config.triggers.slice(), missing: [] },
|
|
1799
2115
|
integrityCheck: 'skipped',
|
|
1800
2116
|
sourceCount: null,
|
|
1801
2117
|
indexCount: null,
|
|
@@ -1806,12 +2122,12 @@ export class SqliteStore {
|
|
|
1806
2122
|
report.tables[table] = info
|
|
1807
2123
|
if (!info.exists) {
|
|
1808
2124
|
missingTable = true
|
|
1809
|
-
info.triggers.missing =
|
|
1810
|
-
report.errors.push(`${table}: the FTS index table is missing although schema
|
|
2125
|
+
info.triggers.missing = config.triggers.slice()
|
|
2126
|
+
report.errors.push(`${table}: the FTS index table is missing although the current schema expects it`)
|
|
1811
2127
|
continue
|
|
1812
2128
|
}
|
|
1813
2129
|
anyTable = true
|
|
1814
|
-
info.triggers.missing =
|
|
2130
|
+
info.triggers.missing = config.triggers.filter((name) => !existingTriggers.has(name))
|
|
1815
2131
|
if (info.triggers.missing.length) {
|
|
1816
2132
|
inconsistent = true
|
|
1817
2133
|
report.errors.push(`${table}: missing FTS sync trigger(s) ${info.triggers.missing.join(', ')}; rows written without them are not indexed`)
|
|
@@ -1824,7 +2140,7 @@ export class SqliteStore {
|
|
|
1824
2140
|
inconsistent = true
|
|
1825
2141
|
report.errors.push(`${table}: FTS5 integrity-check failed: ${String(error?.message ?? error).slice(0, 200)}`)
|
|
1826
2142
|
}
|
|
1827
|
-
try { info.sourceCount = Number(this.db.prepare(`SELECT COUNT(*) AS n FROM ${
|
|
2143
|
+
try { info.sourceCount = Number(this.db.prepare(`SELECT COUNT(*) AS n FROM ${sourceTable}`).get()?.n ?? 0) } catch (error) { report.errors.push(`${table}: cannot count ${sourceTable}: ${String(error?.message ?? error).slice(0, 120)}`) }
|
|
1828
2144
|
try { info.indexCount = Number(this.db.prepare(`SELECT COUNT(*) AS n FROM ${table}_docsize`).get()?.n ?? 0) } catch (error) { info.indexCount = null }
|
|
1829
2145
|
if (info.sourceCount != null && info.indexCount != null && info.sourceCount !== info.indexCount) {
|
|
1830
2146
|
info.countMismatch = true
|
|
@@ -1834,7 +2150,7 @@ export class SqliteStore {
|
|
|
1834
2150
|
if (info.sourceCount && info.sourceCount > 0) {
|
|
1835
2151
|
try {
|
|
1836
2152
|
const limitN = Math.max(1, Math.min(Number(sampleSize) || 0, 1000))
|
|
1837
|
-
const sampleIds = this.db.prepare(`SELECT rowid AS id FROM ${
|
|
2153
|
+
const sampleIds = this.db.prepare(`SELECT rowid AS id FROM ${sourceTable} ORDER BY rowid LIMIT ?`).all(limitN).map((row) => row.id)
|
|
1838
2154
|
if (sampleIds.length) {
|
|
1839
2155
|
const indexed = new Set(this.db.prepare(`SELECT id FROM ${table}_docsize WHERE id IN (${sampleIds.map(() => '?').join(',')})`).all(...sampleIds).map((row) => row.id))
|
|
1840
2156
|
info.sample.checked = sampleIds.length
|
|
@@ -1845,7 +2161,7 @@ export class SqliteStore {
|
|
|
1845
2161
|
}
|
|
1846
2162
|
const indexIds = this.db.prepare(`SELECT id FROM ${table}_docsize ORDER BY id LIMIT ?`).all(limitN).map((row) => row.id)
|
|
1847
2163
|
if (indexIds.length) {
|
|
1848
|
-
const sourceIds = new Set(this.db.prepare(`SELECT rowid AS id FROM ${
|
|
2164
|
+
const sourceIds = new Set(this.db.prepare(`SELECT rowid AS id FROM ${sourceTable} WHERE rowid IN (${indexIds.map(() => '?').join(',')})`).all(...indexIds).map((row) => row.id))
|
|
1849
2165
|
info.sample.extraInIndex = indexIds.filter((id) => !sourceIds.has(id))
|
|
1850
2166
|
if (info.sample.extraInIndex.length) {
|
|
1851
2167
|
inconsistent = true
|
|
@@ -1860,13 +2176,14 @@ export class SqliteStore {
|
|
|
1860
2176
|
}
|
|
1861
2177
|
report.triggers.missing = FTS_TRIGGER_NAMES.filter((name) => !existingTriggers.has(name))
|
|
1862
2178
|
if (!anyTable) {
|
|
1863
|
-
|
|
2179
|
+
const degraded = this.ftsWarning || this.kbFtsWarning
|
|
2180
|
+
if (degraded) {
|
|
1864
2181
|
report.state = 'degraded'
|
|
1865
|
-
report.warning = `FTS5 is unavailable/degraded (${
|
|
2182
|
+
report.warning = `FTS5 is unavailable/degraded (${degraded}); search uses LIKE only`
|
|
1866
2183
|
} else {
|
|
1867
2184
|
report.state = 'missing'
|
|
1868
2185
|
report.ok = false
|
|
1869
|
-
report.errors.push('no FTS index tables exist although schema
|
|
2186
|
+
report.errors.push('no FTS index tables exist although the current schema expects them')
|
|
1870
2187
|
}
|
|
1871
2188
|
} else if (inconsistent || missingTable) {
|
|
1872
2189
|
report.state = 'inconsistent'
|
|
@@ -2293,6 +2610,63 @@ CREATE TABLE IF NOT EXISTS task_runs (
|
|
|
2293
2610
|
return this.db.prepare(sql).all(...params, limit).map(mapDeliveryRow)
|
|
2294
2611
|
}
|
|
2295
2612
|
|
|
2613
|
+
/**
|
|
2614
|
+
* Delivery rows for one receipt subject (0.3.2): row ownership by the
|
|
2615
|
+
* subject's current matter ids OR the generation-time client/project
|
|
2616
|
+
* snapshot columns. The union is deliberate — a delivery recorded for the
|
|
2617
|
+
* client stays scoped to it even after the matter is re-linked, and a row
|
|
2618
|
+
* whose lineage is missing is found through its matter.
|
|
2619
|
+
*
|
|
2620
|
+
* @param {{clientId?: string|null, projectId?: string|null, matterIds?: string[], limit?: number}} [scope]
|
|
2621
|
+
*/
|
|
2622
|
+
listDeliveriesForScope({ clientId = null, projectId = null, matterIds = [], limit = 100 } = {}) {
|
|
2623
|
+
const clauses = []
|
|
2624
|
+
const params = []
|
|
2625
|
+
const ids = Array.isArray(matterIds) ? matterIds.filter((id) => typeof id === 'string' && id) : []
|
|
2626
|
+
if (ids.length) {
|
|
2627
|
+
clauses.push('matter_id IN (' + ids.map(() => '?').join(',') + ')')
|
|
2628
|
+
params.push(...ids)
|
|
2629
|
+
}
|
|
2630
|
+
if (clientId !== null && clientId !== undefined && clientId !== '') {
|
|
2631
|
+
clauses.push('client_id = ?')
|
|
2632
|
+
params.push(clientId)
|
|
2633
|
+
}
|
|
2634
|
+
if (projectId !== null && projectId !== undefined && projectId !== '') {
|
|
2635
|
+
clauses.push('project_id = ?')
|
|
2636
|
+
params.push(projectId)
|
|
2637
|
+
}
|
|
2638
|
+
if (!clauses.length) return []
|
|
2639
|
+
const sql = 'SELECT * FROM deliveries WHERE (' + clauses.join(' OR ') + ') ORDER BY created_at DESC, rowid DESC LIMIT ?'
|
|
2640
|
+
return this.db.prepare(sql).all(...params, limit).map(mapDeliveryRow)
|
|
2641
|
+
}
|
|
2642
|
+
|
|
2643
|
+
/**
|
|
2644
|
+
* Count delivery rows for one receipt subject without loading files.
|
|
2645
|
+
* Mirrors listDeliveriesForScope's scope predicate; receipt packs themselves
|
|
2646
|
+
* are excluded so a new receipt cannot count older receipt packs.
|
|
2647
|
+
*/
|
|
2648
|
+
countDeliveriesForScope({ clientId = null, projectId = null, matterIds = [], excludeReceiptPacks = true } = {}) {
|
|
2649
|
+
const scopeClauses = []
|
|
2650
|
+
const params = []
|
|
2651
|
+
const ids = Array.isArray(matterIds) ? matterIds.filter((id) => typeof id === 'string' && id) : []
|
|
2652
|
+
if (ids.length) {
|
|
2653
|
+
scopeClauses.push('matter_id IN (' + ids.map(() => '?').join(',') + ')')
|
|
2654
|
+
params.push(...ids)
|
|
2655
|
+
}
|
|
2656
|
+
if (clientId !== null && clientId !== undefined && clientId !== '') {
|
|
2657
|
+
scopeClauses.push('client_id = ?')
|
|
2658
|
+
params.push(clientId)
|
|
2659
|
+
}
|
|
2660
|
+
if (projectId !== null && projectId !== undefined && projectId !== '') {
|
|
2661
|
+
scopeClauses.push('project_id = ?')
|
|
2662
|
+
params.push(projectId)
|
|
2663
|
+
}
|
|
2664
|
+
if (!scopeClauses.length) return 0
|
|
2665
|
+
const clauses = ['(' + scopeClauses.join(' OR ') + ')']
|
|
2666
|
+
if (excludeReceiptPacks) clauses.push("COALESCE(tool, '') != 'receipt-pack'")
|
|
2667
|
+
return this.db.prepare('SELECT COUNT(*) AS n FROM deliveries WHERE ' + clauses.join(' AND ')).get(...params).n
|
|
2668
|
+
}
|
|
2669
|
+
|
|
2296
2670
|
getMeta(key) {
|
|
2297
2671
|
return this.db.prepare('SELECT value FROM meta WHERE key = ?').get(key)?.value
|
|
2298
2672
|
}
|
|
@@ -3789,6 +4163,84 @@ CREATE TABLE IF NOT EXISTS task_runs (
|
|
|
3789
4163
|
return this._project('SELECT DISTINCT creditor FROM commitments', [], (row) => row[0])
|
|
3790
4164
|
}
|
|
3791
4165
|
|
|
4166
|
+
/**
|
|
4167
|
+
* MCP narrow commitment read model (0.3.2 P2-1).
|
|
4168
|
+
*
|
|
4169
|
+
* The four MCP list tools (today / commit_ls / who / remind) previously mapped
|
|
4170
|
+
* every commitment entity (`SELECT *` + JSON.parse per row + a Ledger Map) just
|
|
4171
|
+
* to drop almost all rows before serialization. This projection selects only
|
|
4172
|
+
* the eight columns those read models need and applies the per-item clearance
|
|
4173
|
+
* gate here, in rowid (insertion) order:
|
|
4174
|
+
*
|
|
4175
|
+
* - `WHERE status = 'open'` when `all` is false; every status otherwise;
|
|
4176
|
+
* - `NOT INDEXED` keeps the natural table/rowid order, which is exactly the
|
|
4177
|
+
* order `loadCommitments()` (+ its `ORDER BY rowid`) used to produce, so the
|
|
4178
|
+
* MCP output stays byte-identical;
|
|
4179
|
+
* - `hidden = secrecy > clearance`. The secrecy rule mirrors
|
|
4180
|
+
* `lib/evidence.js` `evidenceRecordSecrecy()` exactly: no matter id -> 0, a
|
|
4181
|
+
* matter id missing from the matters table -> Infinity, a null/non-finite
|
|
4182
|
+
* matter secrecy -> 0/Infinity (fail closed);
|
|
4183
|
+
* - `reminder_offsets` is parsed only when it is a JSON array, matching
|
|
4184
|
+
* `mapCommitmentRow` (remind needs the custom offsets).
|
|
4185
|
+
*
|
|
4186
|
+
* `total` is the number of scanned rows (visible + hidden); subtract
|
|
4187
|
+
* `protectedExcluded` for the visible count. No write, no schema/index change.
|
|
4188
|
+
*
|
|
4189
|
+
* @param {{ all?: boolean, clearance?: number }} [options]
|
|
4190
|
+
* @returns {{rows: Array<object>, total: number, protectedExcluded: number}}
|
|
4191
|
+
*/
|
|
4192
|
+
projectCommitmentReadModel({ all = false, clearance = 0 } = {}) {
|
|
4193
|
+
const sql = all
|
|
4194
|
+
? 'SELECT id, debtor, creditor, action, due_at, status, matter_id, reminder_offsets FROM commitments NOT INDEXED'
|
|
4195
|
+
: "SELECT id, debtor, creditor, action, due_at, status, matter_id, reminder_offsets FROM commitments NOT INDEXED WHERE status = 'open'"
|
|
4196
|
+
const statement = this.db.prepare(sql)
|
|
4197
|
+
if (typeof statement.setReturnArrays === 'function') statement.setReturnArrays(true)
|
|
4198
|
+
const raw = statement.all()
|
|
4199
|
+
const matterRows = this.db.prepare('SELECT id, secrecy FROM matters').all()
|
|
4200
|
+
const secrecyById = new Map()
|
|
4201
|
+
for (const matter of matterRows) secrecyById.set(matter.id, matter.secrecy)
|
|
4202
|
+
const numericClearance = Number(clearance)
|
|
4203
|
+
const level = Number.isFinite(numericClearance) && numericClearance >= 0 ? numericClearance : 0
|
|
4204
|
+
const rows = []
|
|
4205
|
+
let protectedExcluded = 0
|
|
4206
|
+
for (const row of raw) {
|
|
4207
|
+
const matterId = row[6]
|
|
4208
|
+
let secrecy = 0
|
|
4209
|
+
if (typeof matterId === 'string' && matterId) {
|
|
4210
|
+
if (!secrecyById.has(matterId)) {
|
|
4211
|
+
secrecy = Number.POSITIVE_INFINITY
|
|
4212
|
+
} else {
|
|
4213
|
+
const matterLevel = Number(secrecyById.get(matterId))
|
|
4214
|
+
secrecy = Number.isFinite(matterLevel) ? matterLevel : Number.POSITIVE_INFINITY
|
|
4215
|
+
}
|
|
4216
|
+
}
|
|
4217
|
+
const hidden = secrecy > level
|
|
4218
|
+
if (hidden) protectedExcluded += 1
|
|
4219
|
+
let offsets
|
|
4220
|
+
if (row[7] != null) {
|
|
4221
|
+
try {
|
|
4222
|
+
const parsed = JSON.parse(row[7])
|
|
4223
|
+
if (Array.isArray(parsed)) offsets = parsed
|
|
4224
|
+
} catch {
|
|
4225
|
+
// Corrupt JSON degrades to "no custom offsets", exactly like mapCommitmentRow.
|
|
4226
|
+
}
|
|
4227
|
+
}
|
|
4228
|
+
rows.push({
|
|
4229
|
+
id: row[0],
|
|
4230
|
+
debtor: row[1],
|
|
4231
|
+
creditor: row[2],
|
|
4232
|
+
action: row[3],
|
|
4233
|
+
dueAt: row[4] ?? undefined,
|
|
4234
|
+
status: row[5],
|
|
4235
|
+
matterId: row[6] ?? undefined,
|
|
4236
|
+
...(offsets === undefined ? {} : { reminderOffsets: offsets }),
|
|
4237
|
+
hidden,
|
|
4238
|
+
})
|
|
4239
|
+
}
|
|
4240
|
+
return { rows, total: rows.length, protectedExcluded }
|
|
4241
|
+
}
|
|
4242
|
+
|
|
4243
|
+
|
|
3792
4244
|
loadPreferences({ status, kind } = {}) {
|
|
3793
4245
|
const where = []
|
|
3794
4246
|
const params = []
|