@wirecat/cli-core 0.390.0 → 0.392.0

package/dist/store/sqlite/lucene.js

@@ -4,7 +4,7 @@ import { isStemmed } from "../../search/lucene/resolved.js";
 import { exhausted, QUERY_LIMITS, searchError } from "../../search/lucene/types.js";
 import { normalize } from "../normalize.js";
 import { emailHitsByPk } from "./emails.js";
-import { combine, FIELD_COMPILERS, quoted, SPEECH, } from "./lucene-fields.js";
+import { combine, FIELD_COMPILERS, ftsMatch, SPEECH, } from "./lucene-fields.js";
 import { hitsByPk } from "./search.js";
 import { read } from "./transaction.js";
 export { prefixOf } from "./lucene-fields.js";
@@ -83,11 +83,12 @@ const compileQuery = (context, execution, boundedAttachments = false) => {
         params: [match],
     });
     /** A bare word also matches a transcript, a file's text and a word of a file's name. */
-    const everywhere = (node, text, fragment) => {
+    const everywhere = (node, fragment) => {
         if (!node.implicit || fragment.sql === "0")
             return fragment;
-        const files = attachmentWords(quoted(text), "any");
-        const elsewhere = { sql: `${files.sql} UNION ${namedSql}`, params: [...files.params, quoted(text)] };
+        const match = ftsMatch(node);
+        const files = attachmentWords(match, "any");
+        const elsewhere = { sql: `${files.sql} UNION ${namedSql}`, params: [...files.params, match] };
         return {
             ...fragment,
             sql: `(${fragment.sql} OR m.id IN (${elsewhere.sql}))`,
@@ -285,10 +286,10 @@ const compileQuery = (context, execution, boundedAttachments = false) => {
     }
     // Among hits the message text does not hold: a transcript, then a file's name, then a file's text. bm25 scores
     // of different indexes do not compare, and a long spreadsheet must not crowd out what was said.
-    const places = [...fragments.values()].flatMap(({ node, fragment }) => fragment.elsewhere && fragment.fts ? [normalize(node.value)] : []);
+    const places = [...fragments.values()].flatMap(({ node, fragment }) => fragment.elsewhere && fragment.fts ? [ftsMatch(node)] : []);
     if (places.length && !execution.newest) {
         ranked.sql.push(`s(id, tier) AS MATERIALIZED (SELECT att.attachable_id, min(CASE WHEN ${SPEECH} THEN 1 ELSE 3 END) FROM attachments att WHERE att.attachable_type='${corpus.item}' AND att.id IN (SELECT rowid FROM attachment_words WHERE attachment_words MATCH ?) GROUP BY att.attachable_id UNION ALL SELECT attachable_id, 2 FROM (${namedSql}))`);
-        ranked.params.push(places.map((text) => quoted(text)).join(" OR "), places.map((text) => quoted(text)).join(" OR "));
+        ranked.params.push(places.join(" OR "), places.join(" OR "));
     }
     const placeTier = places.length && !execution.newest ? "coalesce((SELECT min(tier) FROM s WHERE s.id=m.id), 4)," : "";
     if (exactTier) {
@@ -342,10 +343,10 @@ const compileQuery = (context, execution, boundedAttachments = false) => {
         if (!placeLeaves.length || !pks.length || found.length !== pks.length)
             return found;
         const keys = JSON.stringify(pks);
-        const texts = placeLeaves.map(({ node }) => normalize(node.value)).filter((text) => /[\p{L}\p{N}]/u.test(text));
-        if (!texts.length)
+        const wordy = placeLeaves.filter(({ node }) => /[\p{L}\p{N}]/u.test(normalize(node.value)));
+        if (!wordy.length)
             return found;
-        const match = texts.map((text) => quoted(text)).join(" OR ");
+        const match = wordy.map(({ node }) => ftsMatch(node)).join(" OR ");
         const implicit = placeLeaves.some(({ node }) => node.implicit);
         const inText = new Set(implicit
             ? database
@@ -369,7 +370,6 @@ const compileQuery = (context, execution, boundedAttachments = false) => {
                 .prepare(`SELECT att.attachable_id AS id, att.name AS name FROM attachments att WHERE att.attachable_type='${corpus.item}' AND att.attachable_id IN (SELECT value FROM json_each(?)) AND att.name IS NOT NULL ORDER BY att.position`)
                 .all(keys)
             : [];
-        const wanted = texts.map((text) => text.split(/[^\p{L}\p{N}]+/u).filter(Boolean));
         return found.map((hit, index) => {
             const pk = pks[index];
             const transcript = files.find((row) => Number(row.id) === pk && Number(row.speech) === 1);
@@ -378,7 +378,12 @@ const compileQuery = (context, execution, boundedAttachments = false) => {
                 if (Number(row.id) !== pk)
                     return false;
                 const words = normalize(String(row.name)).split(/[^\p{L}\p{N}]+/u);
-                return wanted.some((phrase) => words.some((_, at) => phrase.every((word, offset) => words[at + offset] === word)));
+                return wordy.some(({ node }) => {
+                    const phrase = normalize(node.value).match(/[\p{L}\p{N}]+/gu) ?? [];
+                    if (node.slop !== undefined)
+                        return phrase.every((word) => words.includes(word));
+                    return words.some((_, at) => phrase.every((word, offset) => words[at + offset] === word));
+                });
             });
             const place = inText.has(pk)
                 ? ["text", hit.text]

package/dist/store/sqlite/meeting-chat-lines.d.ts

@@ -0,0 +1,3 @@
+import { type MeetingChatLinesStore } from "../../meetings/index.js";
+import type { StoreContext } from "./open.js";
+export declare const meetingChatLinesOver: ({ database }: StoreContext) => MeetingChatLinesStore;

package/dist/store/sqlite/meeting-chat-lines.js

@@ -0,0 +1,34 @@
+import { CliError } from "@wirecat/cli-base";
+import { CHAT_MESSAGES_READ } from "../../meetings/index.js";
+import { checkMeetingCancelled } from "./meeting-operation.js";
+import { positiveReadInteger } from "./meeting-reads.js";
+import { meetingSelectionSql } from "./meeting-selection.js";
+import { read } from "./transaction.js";
+export const meetingChatLinesOver = ({ database }) => ({
+    async meetingChatLines(input) {
+        checkMeetingCancelled(input.signal);
+        positiveReadInteger(input.accountId, "accountId");
+        const most = input.maxMessages ?? CHAT_MESSAGES_READ;
+        if (!Number.isSafeInteger(most) || most < 1 || most > CHAT_MESSAGES_READ)
+            throw new CliError("validation_error", `read 1 through ${CHAT_MESSAGES_READ} chat messages`);
+        return read(database, () => {
+            const { clauses, params } = meetingSelectionSql(input);
+            const rows = database
+                .prepare(`SELECT c.meeting_id, m.account_id, m.title, c.sent_at, coalesce(c.sender_name, p.display_name) AS sender, c.text
+           FROM meetings m JOIN meeting_chat_messages c ON c.meeting_id = m.id
+           LEFT JOIN meeting_participants p ON p.id = c.sender_participant_id
+           WHERE ${clauses.join(" AND ")} ORDER BY c.sent_at, c.id LIMIT ?`)
+                .all(...params, most + 1);
+            if (rows.length > most)
+                throw new CliError("validation_error", `the selection has more than ${most} chat messages; narrow it`);
+            return rows.map((row) => ({
+                meetingId: Number(row.meeting_id),
+                accountId: Number(row.account_id),
+                title: row.title === null ? null : String(row.title),
+                sentAt: Number(row.sent_at),
+                sender: row.sender === null ? null : String(row.sender),
+                text: String(row.text),
+            }));
+        });
+    },
+});

package/dist/store/sqlite/meeting-review.d.ts

@@ -0,0 +1,4 @@
+import { type MeetingReviewStore } from "../../meetings/index.js";
+import type { StoreContext } from "./open.js";
+/** Talk and words come from one transcript per meeting, its earliest current one, as in meeting stats. */
+export declare const meetingReviewOver: ({ database }: StoreContext) => MeetingReviewStore;

package/dist/store/sqlite/meeting-review.js

@@ -0,0 +1,97 @@
+import { CliError } from "@wirecat/cli-base";
+import { REVIEW_LIMIT } from "../../meetings/index.js";
+import { fromJson } from "./events.js";
+import { checkMeetingCancelled } from "./meeting-operation.js";
+import { positiveReadInteger } from "./meeting-reads.js";
+import { meetingSelectionSql } from "./meeting-selection.js";
+import { read } from "./transaction.js";
+const number = (value) => (value === null || value === undefined ? null : Number(value));
+const text = (value) => (value === null || value === undefined ? null : String(value));
+/** Talk and words come from one transcript per meeting, its earliest current one, as in meeting stats. */
+export const meetingReviewOver = ({ database }) => ({
+    async meetingReview(input) {
+        checkMeetingCancelled(input.signal);
+        positiveReadInteger(input.accountId, "accountId");
+        const limit = input.limit ?? REVIEW_LIMIT.default;
+        if (!Number.isSafeInteger(limit) || limit < 1 || limit > REVIEW_LIMIT.most)
+            throw new CliError("validation_error", `a review shows 1 through ${REVIEW_LIMIT.most} meetings`);
+        return read(database, () => {
+            const { clauses, params } = meetingSelectionSql(input);
+            const meetings = database
+                .prepare(`SELECT m.id, m.account_id, m.title, m.started_at, m.duration_ms, m.participants_count FROM meetings m
+           WHERE ${clauses.join(" AND ")} ORDER BY m.started_at DESC, m.id DESC LIMIT ?`)
+                .all(...params, limit + 1);
+            const shown = meetings.slice(0, limit);
+            const items = new Map();
+            for (const row of shown)
+                items.set(Number(row.id), {
+                    meetingId: Number(row.id),
+                    accountId: Number(row.account_id),
+                    title: text(row.title),
+                    startedAt: number(row.started_at),
+                    durationMs: number(row.duration_ms),
+                    participantsCount: number(row.participants_count),
+                    projects: [],
+                    transcript: null,
+                    speakers: [],
+                    summary: null,
+                });
+            if (items.size) {
+                const ids = [...items.keys()];
+                const marks = ids.map(() => "?").join(",");
+                checkMeetingCancelled(input.signal);
+                const talk = database
+                    .prepare(`WITH chosen AS (SELECT min(t.id) AS id, t.meeting_id FROM meeting_transcripts t
+               WHERE t.meeting_id IN (${marks}) AND t.deleted_at IS NULL AND t.superseded_at IS NULL GROUP BY t.meeting_id)
+             SELECT c.meeting_id, coalesce(person.name, i.name, r.speaker_name, p.display_name) AS speaker,
+               count(*) AS lines, sum(max(r.end_ms - r.start_ms, 0)) AS talk_ms,
+               sum(length(trim(r.text)) - length(replace(trim(r.text), ' ', '')) + (length(trim(r.text)) > 0)) AS words
+             FROM chosen c JOIN meeting_transcript_rows r ON r.meeting_transcript_id = c.id
+             LEFT JOIN meeting_participants p ON p.id = r.speaker_participant_id
+             LEFT JOIN identities i ON i.id = p.identity_id
+             LEFT JOIN identity_links l ON l.identity_id = p.identity_id
+             LEFT JOIN persons person ON person.id = l.person_id
+             GROUP BY c.meeting_id, speaker ORDER BY c.meeting_id, talk_ms DESC`)
+                    .all(...ids);
+                for (const row of talk) {
+                    const item = items.get(Number(row.meeting_id));
+                    item.transcript ??= { lines: 0, words: 0 };
+                    item.transcript.lines += Number(row.lines);
+                    item.transcript.words += Number(row.words);
+                    if (item.speakers.length < 3)
+                        item.speakers.push({ name: text(row.speaker), talkMs: Number(row.talk_ms) });
+                }
+                const transcribed = database
+                    .prepare(`SELECT DISTINCT meeting_id FROM meeting_transcripts
+             WHERE meeting_id IN (${marks}) AND deleted_at IS NULL AND superseded_at IS NULL`)
+                    .all(...ids);
+                for (const row of transcribed) {
+                    const item = items.get(Number(row.meeting_id));
+                    item.transcript ??= { lines: 0, words: 0 };
+                }
+                const summaries = database
+                    .prepare(`SELECT meeting_id, source, title, overview, next_steps, doc_url FROM meeting_summaries
+             WHERE meeting_id IN (${marks}) ORDER BY coalesce(external_updated_at, updated_at) DESC, id DESC`)
+                    .all(...ids);
+                for (const row of summaries) {
+                    const item = items.get(Number(row.meeting_id));
+                    item.summary ??= {
+                        source: String(row.source),
+                        title: text(row.title),
+                        overview: text(row.overview),
+                        nextSteps: fromJson(row.next_steps) ?? [],
+                        url: text(row.doc_url),
+                    };
+                }
+                const projects = database
+                    .prepare(`SELECT l.from_id, p.name FROM links l JOIN projects p ON p.id = l.to_id
+             WHERE l.from_type = 'meeting' AND l.from_id IN (${marks}) AND l.to_type = 'project' AND l.kind = 'member-of'
+               AND l.anchor IS NULL AND l.confirmed = 1 AND p.account_id IS NULL AND p.deleted_at IS NULL ORDER BY p.name`)
+                    .all(...ids);
+                for (const row of projects)
+                    items.get(Number(row.from_id)).projects.push(String(row.name));
+            }
+            return { items: [...items.values()], hasMore: meetings.length > limit };
+        });
+    },
+});

package/dist/store/sqlite/meetings.d.ts

@@ -1,4 +1,4 @@
-import type { Attachment, MeetingContextStore, MeetingParticipantSearchStore, MeetingPreparationContextStore, MeetingPreparationKnowledgeStore, MeetingPreparationNotesStore, MeetingRelatedHistoryStore, MeetingSearchIndexStore, MeetingSearchStore, MeetingSeriesHistoryStore, MeetingStatsStore, MeetingStore, MeetingTranscriptReceiptStore, MeetingTranscriptStore, NewRecord } from "../../meetings/index.js";
+import type { Attachment, MeetingChatLinesStore, MeetingContextStore, MeetingParticipantSearchStore, MeetingPreparationContextStore, MeetingPreparationKnowledgeStore, MeetingPreparationNotesStore, MeetingRelatedHistoryStore, MeetingReviewStore, MeetingSearchIndexStore, MeetingSearchStore, MeetingSeriesHistoryStore, MeetingStatsStore, MeetingStore, MeetingTranscriptReceiptStore, MeetingTranscriptStore, NewRecord } from "../../meetings/index.js";
 import type { CacheDatabase } from "../driver.js";
 import { type MeetingContextReadCapabilities, type MeetingReadCapabilities } from "./meeting-reads.js";
 import type { StoreContext } from "./open.js";
@@ -11,7 +11,7 @@ export declare const saveAttachment: (database: CacheDatabase, type: string, par
 export declare const drainMeetingIndex: (database: CacheDatabase, batch?: number) => number;
 /** Each word of the query, as a prefix, all required. */
 export declare const wordsQuery: (query: string) => string | null;
-export type StoredMeetings = MeetingStore & MeetingPreparationContextStore & MeetingPreparationKnowledgeStore & MeetingPreparationNotesStore & MeetingRelatedHistoryStore & MeetingSeriesHistoryStore & MeetingTranscriptStore & MeetingReadCapabilities & MeetingContextReadCapabilities & MeetingParticipantSearchStore & MeetingSearchStore & MeetingTranscriptReceiptStore & MeetingContextStore & MeetingSearchIndexStore & MeetingStatsStore;
+export type StoredMeetings = MeetingStore & MeetingPreparationContextStore & MeetingPreparationKnowledgeStore & MeetingPreparationNotesStore & MeetingRelatedHistoryStore & MeetingSeriesHistoryStore & MeetingTranscriptStore & MeetingReadCapabilities & MeetingContextReadCapabilities & MeetingParticipantSearchStore & MeetingSearchStore & MeetingTranscriptReceiptStore & MeetingContextStore & MeetingSearchIndexStore & MeetingStatsStore & MeetingReviewStore & MeetingChatLinesStore;
 /** The `MeetingStore` of `src/meetings/` over the shared store. */
 export declare const meetingStoreOver: (context: StoreContext) => StoredMeetings;
 export {};

package/dist/store/sqlite/meetings.js

@@ -5,6 +5,7 @@ import * as eventQueries from "./events.js";
 import { fromJson, int, page, str, toJson } from "./events.js";
 import { folded } from "./folded.js";
 import { meetingIdentityPk } from "./identities.js";
+import { meetingChatLinesOver } from "./meeting-chat-lines.js";
 import { meetingContextLinksOver } from "./meeting-context-links.js";
 import { meetingHitIndexOver } from "./meeting-hit-index.js";
 import { meetingHitSearchOver } from "./meeting-hit-search.js";
@@ -15,6 +16,7 @@ import { meetingPreparationKnowledgeOver } from "./meeting-preparation-knowledge
 import { meetingPreparationNotesOver } from "./meeting-preparation-notes.js";
 import { meetingReadsOver } from "./meeting-reads.js";
 import { meetingRelatedHistoryOver } from "./meeting-related-history.js";
+import { meetingReviewOver } from "./meeting-review.js";
 import { meetingSeriesHistoryOver } from "./meeting-series-history.js";
 import { meetingStatsOver } from "./meeting-stats.js";
 import { invalidateMeetingStems } from "./meeting-stem-state.js";
@@ -573,6 +575,8 @@ export const meetingStoreOver = (context) => {
         ...meetingRelatedHistoryOver(context),
         ...meetingHitSearchOver(context),
         ...meetingStatsOver(context),
+        ...meetingReviewOver(context),
+        ...meetingChatLinesOver(context),
         ...meetingParticipantSearchOver(context),
         ...meetingHitIndexOver(context),
         ...meetingContextLinksOver(context),

package/dist/store/sqlite/mention-drain.js

@@ -1,6 +1,7 @@
 import { findMentions, MENTION_KINDS, MENTION_VERSIONS, oneTrackedLinkPerSite, telegramTarget, } from "../../domain/mentions.js";
 import { findNames, findUnknownNames, nameIndex } from "../../domain/person-names.js";
 import { createStemmer } from "../../search/stem.js";
+import { speech } from "./lucene-fields.js";
 import { storePhoneCountries } from "./phone-countries.js";
 import { placeOf } from "./place-key.js";
 /** Mentions a store write finds before it commits: an older binary may have queued thousands. */
@@ -98,10 +99,10 @@ export const SOURCES = {
     attachment: {
         count: "SELECT count(*) AS n FROM attachments a JOIN messages m ON m.id = a.attachable_id " +
             "WHERE a.attachable_type = 'message' AND a.text IS NOT NULL AND a.text <> '' AND m.deleted_at IS NULL",
-        page: "SELECT a.id, a.text, CASE WHEN coalesce(a.extractor, '') LIKE 'speech:%' THEN 'transcript' ELSE 'file_text' END AS field, " +
+        page: `SELECT a.id, a.text, CASE WHEN ${speech("a")} THEN 'transcript' ELSE 'file_text' END AS field, ` +
             "m.chat_id FROM attachments a JOIN messages m ON m.id = a.attachable_id WHERE a.id > ? AND a.attachable_type = 'message' " +
             "AND a.text IS NOT NULL AND a.text <> '' AND m.deleted_at IS NULL ORDER BY a.id LIMIT ?",
-        one: "SELECT a.text, CASE WHEN coalesce(a.extractor, '') LIKE 'speech:%' THEN 'transcript' ELSE 'file_text' END AS field, " +
+        one: `SELECT a.text, CASE WHEN ${speech("a")} THEN 'transcript' ELSE 'file_text' END AS field, ` +
             "m.chat_id FROM attachments a JOIN messages m ON m.id = a.attachable_id WHERE a.id = ? AND a.attachable_type = 'message' " +
             "AND a.text IS NOT NULL AND m.deleted_at IS NULL",
     },

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@wirecat/cli-core",
-  "version": "0.390.0",
+  "version": "0.392.0",
   "description": "The messenger-neutral half of a messaging CLI: domain model, message locators, rendering, name resolution, the SQLite seam, the command skeleton and shared read commands, the send guard, run records and the message store",
   "license": "Apache-2.0",
   "author": "Viacheslav Ptsarev",