@wirecat/cli-core 0.387.0 → 0.389.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +25 -1
- package/dist/cli/messenger/store-maintenance-command.d.ts +1 -1
- package/dist/meetings/index.d.ts +1 -0
- package/dist/meetings/index.js +1 -0
- package/dist/meetings/stats.d.ts +59 -0
- package/dist/meetings/stats.js +95 -0
- package/dist/search/question-plan.d.ts +5 -0
- package/dist/search/question-plan.js +11 -0
- package/dist/services/index.d.ts +2 -0
- package/dist/services/index.js +2 -0
- package/dist/services/meeting-combined.d.ts +95 -0
- package/dist/services/meeting-combined.js +163 -0
- package/dist/services/meeting-hit-search.d.ts +26 -0
- package/dist/services/meeting-hit-search.js +5 -1
- package/dist/services/meeting-semantic.d.ts +1 -0
- package/dist/services/meeting-semantic.js +1 -1
- package/dist/services/meeting-stats.d.ts +13 -0
- package/dist/services/meeting-stats.js +12 -0
- package/dist/store/sqlite/meeting-hit-search.js +2 -38
- package/dist/store/sqlite/meeting-selection.d.ts +8 -0
- package/dist/store/sqlite/meeting-selection.js +42 -0
- package/dist/store/sqlite/meeting-stats.d.ts +7 -0
- package/dist/store/sqlite/meeting-stats.js +55 -0
- package/dist/store/sqlite/meeting-vectors.d.ts +17 -0
- package/dist/store/sqlite/meeting-vectors.js +66 -0
- package/dist/store/sqlite/meetings.d.ts +2 -2
- package/dist/store/sqlite/meetings.js +2 -0
- package/package.json +1 -1
package/CHANGELOG.md
CHANGED
|
@@ -5,7 +5,31 @@ Notable changes to `@wirecat/cli-core` (`@wirecat/cli-messaging` up to 0.261.0,
|
|
|
5
5
|
minor version may break callers, and says how under "Changed — may break callers". `pnpm docs:check` checks the
|
|
6
6
|
shape of this file.
|
|
7
7
|
|
|
8
|
-
##
|
|
8
|
+
## 0.389.0 — 22.10.2026
|
|
9
|
+
|
|
10
|
+
### Added
|
|
11
|
+
|
|
12
|
+
- **`readMeetingStats`: talk time per speaker and meetings per week.** Over the meetings search would read, with the
|
|
13
|
+
same filters: per speaker (a linked person, else the identity, else the transcript's name) lines, words, talk time,
|
|
14
|
+
meetings and share of talk; per week (Monday in a chosen zone) meetings and talk time. One transcript per meeting,
|
|
15
|
+
so a local transcript beside the provider's does not count twice. `meetings.meetingStats` on the store and
|
|
16
|
+
`aggregateMeetingStats` for rows a host already has.
|
|
17
|
+
|
|
18
|
+
## 0.388.0 — 22.10.2026
|
|
19
|
+
|
|
20
|
+
### Added
|
|
21
|
+
|
|
22
|
+
- **`searchMeetingsCombined`: meeting search by words and meaning in one ranking.** Word hits and passage vectors of
|
|
23
|
+
one model are merged by reciprocal rank fusion over one selection of meetings: the date, meeting, series, event,
|
|
24
|
+
participant, person, project and tag filters apply to both. A word hit inside a passage the meaning search found
|
|
25
|
+
joins it (`by: ["words", "meaning"]`) and is cited by its own line. A question («что решили по цене?», "what did we
|
|
26
|
+
decide about pricing?") searches its content words; other text is the meeting query language, and text it refuses
|
|
27
|
+
is asked by meaning alone (`words.refused`). On a tie a line with the words ranks above a passage that only
|
|
28
|
+
resembles the query. The ranking covers
|
|
29
|
+
every vector of the selection up to `maxChunks` (default 10000), read in batches with text loaded only for the
|
|
30
|
+
passages kept; pages continue one ranking (`nextCursor`), bound to the query, filters and model.
|
|
31
|
+
- **`meetingVectors.ranked`** — the nearest passages over the whole selection, with meeting title and start.
|
|
32
|
+
`questionTerms` (`search/question-plan`) — a question's content words, or null.
|
|
9
33
|
|
|
10
34
|
## 0.387.0 — 22.10.2026
|
|
11
35
|
|
|
@@ -2,9 +2,9 @@ import { Command } from "commander";
|
|
|
2
2
|
import type { Messenger } from "./context.js";
|
|
3
3
|
/** `doctor`'s short summary of the store. Must answer for a broken file, so an error is a field. */
|
|
4
4
|
export declare const storeSummary: (env: NodeJS.ProcessEnv) => Promise<{
|
|
5
|
+
error?: undefined;
|
|
5
6
|
path: string;
|
|
6
7
|
exists: boolean;
|
|
7
|
-
error?: undefined;
|
|
8
8
|
} | {
|
|
9
9
|
path: string;
|
|
10
10
|
exists: boolean;
|
package/dist/meetings/index.d.ts
CHANGED
|
@@ -40,6 +40,7 @@ export { createMeetingSeriesInventoryReader } from "./series-inventory.js";
|
|
|
40
40
|
export type { MeetingSeriesCandidate, MeetingSeriesInventoryPage, MeetingSeriesInventoryPort, MeetingSeriesInventoryRead, MeetingSeriesInventoryRequest, MeetingSeriesInventoryResult, } from "./series-inventory-types.js";
|
|
41
41
|
export { validateMeetingSeriesInventoryCursor } from "./series-inventory-validation.js";
|
|
42
42
|
export type { MeetingSource, SourceParticipant } from "./source.js";
|
|
43
|
+
export { aggregateMeetingStats, type MeetingSpeakerStats, type MeetingStats, type MeetingStatsInput, type MeetingStatsRow, type MeetingStatsSelection, type MeetingStatsStore, type MeetingWeekStats, } from "./stats.js";
|
|
43
44
|
export type { ChatInput, ContractCase, EventCandidateFilter, IdentityInput, MeetingDetails, MeetingFilter, MeetingHitSource, MeetingInput, MeetingSave, MeetingStore, NewRecord, ParticipantInput, SearchHit, TranscriptInput, TranscriptRowInput, } from "./store.js";
|
|
44
45
|
export type { MeetingTranscriptReceiptStore, MeetingTranscriptStore, TranscriptAppend, TranscriptAppendReceipt, UnlinkedTranscriptInput, } from "./transcript-store.js";
|
|
45
46
|
export { parseTranscriptText, TranscriptTextError, type TranscriptTextHeader } from "./transcript-text.js";
|
package/dist/meetings/index.js
CHANGED
|
@@ -28,6 +28,7 @@ export { validateMeetingHitSearch } from "./search-validation.js";
|
|
|
28
28
|
export { validateMeetingSeriesHistory, } from "./series-history.js";
|
|
29
29
|
export { createMeetingSeriesInventoryReader } from "./series-inventory.js";
|
|
30
30
|
export { validateMeetingSeriesInventoryCursor } from "./series-inventory-validation.js";
|
|
31
|
+
export { aggregateMeetingStats, } from "./stats.js";
|
|
31
32
|
export { parseTranscriptText, TranscriptTextError } from "./transcript-text.js";
|
|
32
33
|
export { transcribeMeeting, } from "./transcription.js";
|
|
33
34
|
export { createUpcomingMeetingReader } from "./upcoming.js";
|
|
@@ -0,0 +1,59 @@
|
|
|
1
|
+
import type { MeetingSearchOptions } from "./search-store.js";
|
|
2
|
+
/** The meetings the numbers cover: the same filters as meeting search. */
|
|
3
|
+
export type MeetingStatsSelection = Pick<MeetingSearchOptions, "accountId" | "meetingId" | "eventId" | "meetingSeriesId" | "since" | "until" | "participantIdentityId" | "personId" | "projectId" | "tagId" | "signal">;
|
|
4
|
+
export interface MeetingStatsInput extends MeetingStatsSelection {
|
|
5
|
+
/** The zone a week starts on Monday in; UTC when not given. */
|
|
6
|
+
timeZone?: string;
|
|
7
|
+
/** Most transcript lines to read (default 500000); more is an error, never a silent cut. */
|
|
8
|
+
maxRows?: number;
|
|
9
|
+
}
|
|
10
|
+
/** One transcript line as the numbers need it: who said it, when, for how long, in how many words. */
|
|
11
|
+
export interface MeetingStatsRow {
|
|
12
|
+
meetingId: number;
|
|
13
|
+
startedAt: number | null;
|
|
14
|
+
startMs: number;
|
|
15
|
+
endMs: number;
|
|
16
|
+
words: number;
|
|
17
|
+
personId: number | null;
|
|
18
|
+
personName: string | null;
|
|
19
|
+
identityId: number | null;
|
|
20
|
+
identityName: string | null;
|
|
21
|
+
speakerName: string | null;
|
|
22
|
+
}
|
|
23
|
+
export interface MeetingSpeakerStats {
|
|
24
|
+
/** `person:<id>`, `identity:<id>`, `name:<lowercased name>` or `unknown`. */
|
|
25
|
+
key: string;
|
|
26
|
+
name: string | null;
|
|
27
|
+
personId: number | null;
|
|
28
|
+
identityId: number | null;
|
|
29
|
+
meetings: number;
|
|
30
|
+
lines: number;
|
|
31
|
+
words: number;
|
|
32
|
+
talkMs: number;
|
|
33
|
+
/** This speaker's part of all talk time in the selection, 0–1. */
|
|
34
|
+
share: number;
|
|
35
|
+
}
|
|
36
|
+
export interface MeetingWeekStats {
|
|
37
|
+
/** The Monday the week starts on, in the chosen zone: `2026-10-19`. */
|
|
38
|
+
weekStart: string;
|
|
39
|
+
meetings: number;
|
|
40
|
+
talkMs: number;
|
|
41
|
+
}
|
|
42
|
+
export interface MeetingStats {
|
|
43
|
+
timeZone: string;
|
|
44
|
+
meetings: number;
|
|
45
|
+
lines: number;
|
|
46
|
+
words: number;
|
|
47
|
+
talkMs: number;
|
|
48
|
+
speakers: MeetingSpeakerStats[];
|
|
49
|
+
weeks: MeetingWeekStats[];
|
|
50
|
+
}
|
|
51
|
+
export interface MeetingStatsStore {
|
|
52
|
+
meetingStats(input: MeetingStatsInput): Promise<MeetingStats>;
|
|
53
|
+
}
|
|
54
|
+
/**
|
|
55
|
+
* Talk time, lines and words per speaker, and meetings and talk time per week. A speaker is a linked person when the
|
|
56
|
+
* line's participant has one, else the participant's identity, else the name the transcript gives; lines with none
|
|
57
|
+
* are `unknown`. Talk time is each line's end less its start, so overlapping speech counts for everyone speaking.
|
|
58
|
+
*/
|
|
59
|
+
export declare const aggregateMeetingStats: (rows: readonly MeetingStatsRow[], timeZone?: string) => MeetingStats;
|
|
@@ -0,0 +1,95 @@
|
|
|
1
|
+
import { CliError } from "@wirecat/cli-base";
|
|
2
|
+
const zoneOf = (timeZone) => {
|
|
3
|
+
const zone = timeZone ?? "UTC";
|
|
4
|
+
try {
|
|
5
|
+
new Intl.DateTimeFormat("en", { timeZone: zone });
|
|
6
|
+
}
|
|
7
|
+
catch {
|
|
8
|
+
throw new CliError("validation_error", "expected an IANA time zone such as Europe/Madrid");
|
|
9
|
+
}
|
|
10
|
+
return zone;
|
|
11
|
+
};
|
|
12
|
+
const DAYS = ["Mon", "Tue", "Wed", "Thu", "Fri", "Sat", "Sun"];
|
|
13
|
+
const weekStart = (at, format) => {
|
|
14
|
+
const parts = Object.fromEntries(format.formatToParts(new Date(at)).map((part) => [part.type, part.value]));
|
|
15
|
+
const day = Date.UTC(Number(parts.year), Number(parts.month) - 1, Number(parts.day));
|
|
16
|
+
return new Date(day - DAYS.indexOf(parts.weekday ?? "Mon") * 86_400_000).toISOString().slice(0, 10);
|
|
17
|
+
};
|
|
18
|
+
/** Lines with no speaker at all; a constant, since a `key: "…"` literal reads as a permission key to the key test. */
|
|
19
|
+
const UNKNOWN_SPEAKER = "unknown";
|
|
20
|
+
const speakerOf = (row) => row.personId !== null
|
|
21
|
+
? { key: `person:${row.personId}`, name: row.personName ?? row.identityName ?? row.speakerName }
|
|
22
|
+
: row.identityId !== null
|
|
23
|
+
? { key: `identity:${row.identityId}`, name: row.identityName ?? row.speakerName }
|
|
24
|
+
: row.speakerName?.trim()
|
|
25
|
+
? { key: `name:${row.speakerName.trim().toLowerCase()}`, name: row.speakerName.trim() }
|
|
26
|
+
: { key: UNKNOWN_SPEAKER, name: null };
|
|
27
|
+
/**
|
|
28
|
+
* Talk time, lines and words per speaker, and meetings and talk time per week. A speaker is a linked person when the
|
|
29
|
+
* line's participant has one, else the participant's identity, else the name the transcript gives; lines with none
|
|
30
|
+
* are `unknown`. Talk time is each line's end less its start, so overlapping speech counts for everyone speaking.
|
|
31
|
+
*/
|
|
32
|
+
export const aggregateMeetingStats = (rows, timeZone) => {
|
|
33
|
+
const zone = zoneOf(timeZone);
|
|
34
|
+
const format = new Intl.DateTimeFormat("en-US", {
|
|
35
|
+
timeZone: zone,
|
|
36
|
+
year: "numeric",
|
|
37
|
+
month: "2-digit",
|
|
38
|
+
day: "2-digit",
|
|
39
|
+
weekday: "short",
|
|
40
|
+
});
|
|
41
|
+
const speakers = new Map();
|
|
42
|
+
const weeks = new Map();
|
|
43
|
+
const meetings = new Set();
|
|
44
|
+
let lines = 0;
|
|
45
|
+
let words = 0;
|
|
46
|
+
let talkMs = 0;
|
|
47
|
+
for (const row of rows) {
|
|
48
|
+
const talk = Math.max(0, row.endMs - row.startMs);
|
|
49
|
+
meetings.add(row.meetingId);
|
|
50
|
+
lines += 1;
|
|
51
|
+
words += row.words;
|
|
52
|
+
talkMs += talk;
|
|
53
|
+
const who = speakerOf(row);
|
|
54
|
+
const held = speakers.get(who.key) ?? {
|
|
55
|
+
key: who.key,
|
|
56
|
+
name: who.name ?? null,
|
|
57
|
+
personId: row.personId,
|
|
58
|
+
identityId: row.personId === null ? row.identityId : null,
|
|
59
|
+
meetings: 0,
|
|
60
|
+
lines: 0,
|
|
61
|
+
words: 0,
|
|
62
|
+
talkMs: 0,
|
|
63
|
+
share: 0,
|
|
64
|
+
seen: new Set(),
|
|
65
|
+
};
|
|
66
|
+
held.seen.add(row.meetingId);
|
|
67
|
+
held.lines += 1;
|
|
68
|
+
held.words += row.words;
|
|
69
|
+
held.talkMs += talk;
|
|
70
|
+
speakers.set(who.key, held);
|
|
71
|
+
if (row.startedAt !== null) {
|
|
72
|
+
const week = weeks.get(weekStart(row.startedAt, format)) ?? { meetings: new Set(), talkMs: 0 };
|
|
73
|
+
week.meetings.add(row.meetingId);
|
|
74
|
+
week.talkMs += talk;
|
|
75
|
+
weeks.set(weekStart(row.startedAt, format), week);
|
|
76
|
+
}
|
|
77
|
+
}
|
|
78
|
+
return {
|
|
79
|
+
timeZone: zone,
|
|
80
|
+
meetings: meetings.size,
|
|
81
|
+
lines,
|
|
82
|
+
words,
|
|
83
|
+
talkMs,
|
|
84
|
+
speakers: [...speakers.values()]
|
|
85
|
+
.map(({ seen, ...speaker }) => ({
|
|
86
|
+
...speaker,
|
|
87
|
+
meetings: seen.size,
|
|
88
|
+
share: talkMs === 0 ? 0 : Math.round((speaker.talkMs / talkMs) * 1000) / 1000,
|
|
89
|
+
}))
|
|
90
|
+
.sort((a, b) => b.talkMs - a.talkMs || a.key.localeCompare(b.key)),
|
|
91
|
+
weeks: [...weeks.entries()]
|
|
92
|
+
.map(([start, week]) => ({ weekStart: start, meetings: week.meetings.size, talkMs: week.talkMs }))
|
|
93
|
+
.sort((a, b) => a.weekStart.localeCompare(b.weekStart)),
|
|
94
|
+
};
|
|
95
|
+
};
|
|
@@ -1,4 +1,9 @@
|
|
|
1
1
|
export declare const words: (text: string) => RegExpMatchArray | [];
|
|
2
|
+
/**
|
|
3
|
+
* A question's content words — «что решили по цене?» → решили, цене — or null when the text is not a question: it
|
|
4
|
+
* ends with «?» or starts with a question word.
|
|
5
|
+
*/
|
|
6
|
+
export declare const questionTerms: (text: string) => string[] | null;
|
|
2
7
|
export declare function planQuestion(text: string, strict?: boolean): {
|
|
3
8
|
changed: boolean;
|
|
4
9
|
natural: boolean;
|
|
@@ -4,6 +4,17 @@ import { SEARCH_QUERY } from "./lucene/types.js";
|
|
|
4
4
|
export const words = (text) => normalize(text).match(/[\p{L}\p{N}]+/gu) ?? [];
|
|
5
5
|
const scaffolding = new Set(words("what which how many much who where when is are was were do does did can could should we the a an for of to under from in on at has have сколько какой какая какие какое кто кому кем где когда как что во по на в к с у через ли должен должна нужно идёт идет можно может могут"));
|
|
6
6
|
const questionWord = /\b(?:what|which|how|who|where|when|can|could|should)\b|(?:^|\s)(?:сколько|какой|какая|какие|какое|кто|кому|кем|где|когда|как|что|можно|может|могут|ли)(?:\s|$)/iu;
|
|
7
|
+
const QUESTION_STARTS = new Set(words("what which how who whom whose where when why can could should would did do does is are was were will " +
|
|
8
|
+
"что кто кого кому кем где когда как какой какая какое какие какую каким какого каких почему зачем откуда " +
|
|
9
|
+
"куда сколько ли можно разве чей чья чьё чьи"));
|
|
10
|
+
const QUESTION_FILLER = new Set(words("about any did does why whom whose would will это мы вы я они он она о об про ли"));
|
|
11
|
+
/**
|
|
12
|
+
* A question's content words — «что решили по цене?» → решили, цене — or null when the text is not a question: it
|
|
13
|
+
* ends with «?» or starts with a question word.
|
|
14
|
+
*/
|
|
15
|
+
export const questionTerms = (text) => /\?\s*$/u.test(text) || QUESTION_STARTS.has(words(text)[0] ?? "")
|
|
16
|
+
? words(text).filter((word) => !scaffolding.has(word) && !QUESTION_STARTS.has(word) && !QUESTION_FILLER.has(word))
|
|
17
|
+
: null;
|
|
7
18
|
export function planQuestion(text, strict = false) {
|
|
8
19
|
if (strict)
|
|
9
20
|
return {
|
package/dist/services/index.d.ts
CHANGED
|
@@ -67,6 +67,7 @@ export { readMail } from "./mail-search.js";
|
|
|
67
67
|
export { type MeetingHitSearchRequest, refreshMeetingIndex, searchMeetingHits } from "./meeting-hit-search.js";
|
|
68
68
|
export { createMeetingProjectCandidateReader } from "./meeting-project-candidates.js";
|
|
69
69
|
export { readMeetingProjectDiscovery } from "./meeting-project-discovery.js";
|
|
70
|
+
export { type MeetingStatsRequest, readMeetingStats } from "./meeting-stats.js";
|
|
70
71
|
export { chatIdIn, storedChatId } from "./message-resolution.js";
|
|
71
72
|
export type { AroundWindow, FoundMessage, ListWindow, MessagesService, MessageTarget, Pinned, Reacted, SearchFound, SearchQuery, SendRequest, Starred, } from "./messages.js";
|
|
72
73
|
export { DELETE_AT_ONCE, messagesService } from "./messages.js";
|
|
@@ -139,6 +140,7 @@ export { countersService } from "./counters.js";
|
|
|
139
140
|
export { type DocumentTarget, openPictureEmbedderFor, type PictureOptions, removePicture, savePicture, } from "./document-pictures.js";
|
|
140
141
|
export type { ModelChoice } from "./embeddings.js";
|
|
141
142
|
export { createMeetingBatchReader } from "./meeting-batch.js";
|
|
143
|
+
export { type MeetingCombinedHit, type MeetingCombinedRequest, searchMeetingsCombined, } from "./meeting-combined.js";
|
|
142
144
|
export type { MeetingCommunicationCoverage, MeetingCommunicationExcerpt, MeetingCommunications, MeetingCommunicationsPort, MeetingCommunicationsRequest, } from "./meeting-communications-types.js";
|
|
143
145
|
export { createMeetingContextService, type MeetingContextService, type MeetingContextServiceStore, } from "./meeting-context.js";
|
|
144
146
|
export { type MeetingEvidenceCue, type MeetingEvidenceOptions, type MeetingEvidencePacket, readMeetingEvidence, } from "./meeting-evidence.js";
|
package/dist/services/index.js
CHANGED
|
@@ -48,6 +48,7 @@ export { readMail } from "./mail-search.js";
|
|
|
48
48
|
export { refreshMeetingIndex, searchMeetingHits } from "./meeting-hit-search.js";
|
|
49
49
|
export { createMeetingProjectCandidateReader } from "./meeting-project-candidates.js";
|
|
50
50
|
export { readMeetingProjectDiscovery } from "./meeting-project-discovery.js";
|
|
51
|
+
export { readMeetingStats } from "./meeting-stats.js";
|
|
51
52
|
export { chatIdIn, storedChatId } from "./message-resolution.js";
|
|
52
53
|
export { DELETE_AT_ONCE, messagesService } from "./messages.js";
|
|
53
54
|
export { searchStore } from "./messages-archive-search.js";
|
|
@@ -105,6 +106,7 @@ export { GAP_BOUNDS, gapsService } from "./archive-gaps.js";
|
|
|
105
106
|
export { countersService } from "./counters.js";
|
|
106
107
|
export { openPictureEmbedderFor, removePicture, savePicture, } from "./document-pictures.js";
|
|
107
108
|
export { createMeetingBatchReader } from "./meeting-batch.js";
|
|
109
|
+
export { searchMeetingsCombined, } from "./meeting-combined.js";
|
|
108
110
|
export { createMeetingContextService, } from "./meeting-context.js";
|
|
109
111
|
export { readMeetingEvidence, } from "./meeting-evidence.js";
|
|
110
112
|
export { meetingHitContextLimits, readMeetingHitContext, } from "./meeting-hit-context.js";
|
|
@@ -0,0 +1,95 @@
|
|
|
1
|
+
import { type MeetingHitSearchRequest, type SearchHit } from "../meetings/index.js";
|
|
2
|
+
import type { MessageStore } from "../store/store.js";
|
|
3
|
+
import type { MeetingEmbedder } from "./meeting-model.js";
|
|
4
|
+
export interface MeetingCombinedRequest extends MeetingHitSearchRequest {
|
|
5
|
+
/** Most stored passage vectors to compare, 1 through 100000 (default 10000). */
|
|
6
|
+
maxChunks?: number;
|
|
7
|
+
maxRows?: number;
|
|
8
|
+
maxTextBytes?: number;
|
|
9
|
+
}
|
|
10
|
+
export interface MeetingCombinedHit {
|
|
11
|
+
/** A transcript line to cite: the best word hit in the passage, else the passage's first line. */
|
|
12
|
+
ref: string;
|
|
13
|
+
meetingId: number;
|
|
14
|
+
scope: SearchHit["scope"];
|
|
15
|
+
text: string;
|
|
16
|
+
startMs: number | null;
|
|
17
|
+
source: {
|
|
18
|
+
title: string | null;
|
|
19
|
+
startedAt: number | null;
|
|
20
|
+
url: string | null;
|
|
21
|
+
};
|
|
22
|
+
/** Which searches found it: `words`, `meaning` or both. */
|
|
23
|
+
by: ("words" | "meaning")[];
|
|
24
|
+
/** Similarity in meaning, when the meaning search found it. */
|
|
25
|
+
score: number | null;
|
|
26
|
+
/** The passage the meaning search matched, as its first and last line. */
|
|
27
|
+
passage?: {
|
|
28
|
+
first: string;
|
|
29
|
+
last: string;
|
|
30
|
+
};
|
|
31
|
+
match?: SearchHit["match"];
|
|
32
|
+
corrections?: SearchHit["corrections"];
|
|
33
|
+
}
|
|
34
|
+
/**
|
|
35
|
+
* Words and meaning, ranked together by reciprocal rank fusion over one selection of meetings. A question
|
|
36
|
+
* («что решили по цене?») searches its content words; other text is the meeting query language. A word hit inside
|
|
37
|
+
* a passage the meaning search found joins it, so one discussion found both ways ranks first.
|
|
38
|
+
*/
|
|
39
|
+
export declare const searchMeetingsCombined: (store: Pick<MessageStore, "meetings" | "meetingVectors"> & {
|
|
40
|
+
knowledge?: MessageStore["knowledge"];
|
|
41
|
+
}, embedder: MeetingEmbedder, query: string, request: MeetingCombinedRequest) => Promise<{
|
|
42
|
+
items: {
|
|
43
|
+
/** A transcript line to cite: the best word hit in the passage, else the passage's first line. */
|
|
44
|
+
ref: string;
|
|
45
|
+
meetingId: number;
|
|
46
|
+
scope: SearchHit["scope"];
|
|
47
|
+
text: string;
|
|
48
|
+
startMs: number | null;
|
|
49
|
+
source: {
|
|
50
|
+
title: string | null;
|
|
51
|
+
startedAt: number | null;
|
|
52
|
+
url: string | null;
|
|
53
|
+
};
|
|
54
|
+
/** Which searches found it: `words`, `meaning` or both. */
|
|
55
|
+
by: ("words" | "meaning")[];
|
|
56
|
+
/** Similarity in meaning, when the meaning search found it. */
|
|
57
|
+
score: number | null;
|
|
58
|
+
/** The passage the meaning search matched, as its first and last line. */
|
|
59
|
+
passage?: {
|
|
60
|
+
first: string;
|
|
61
|
+
last: string;
|
|
62
|
+
};
|
|
63
|
+
match?: SearchHit["match"];
|
|
64
|
+
corrections?: SearchHit["corrections"];
|
|
65
|
+
}[];
|
|
66
|
+
hasMore: boolean;
|
|
67
|
+
nextCursor: string | null;
|
|
68
|
+
selection: {
|
|
69
|
+
kind: import("../store/index.js").VocabularyKind;
|
|
70
|
+
id: number;
|
|
71
|
+
name: string;
|
|
72
|
+
by: "explicit-link";
|
|
73
|
+
}[];
|
|
74
|
+
words: {
|
|
75
|
+
query: string;
|
|
76
|
+
from?: "question" | undefined;
|
|
77
|
+
refused?: string | undefined;
|
|
78
|
+
};
|
|
79
|
+
model: {
|
|
80
|
+
key: string;
|
|
81
|
+
model: string;
|
|
82
|
+
kind: "local" | "remote";
|
|
83
|
+
dims: number;
|
|
84
|
+
};
|
|
85
|
+
coverage: {
|
|
86
|
+
archiveCompleteness: "unknown";
|
|
87
|
+
indexCompleteness: "unknown";
|
|
88
|
+
indexing: "pending" | "ready";
|
|
89
|
+
/** Every vector of the selection was compared; false when `maxChunks` stopped the scan. */
|
|
90
|
+
meaningScanComplete: boolean;
|
|
91
|
+
scanned: number;
|
|
92
|
+
/** Each side ranks this many at most; pages past them hold nothing more. */
|
|
93
|
+
depth: number;
|
|
94
|
+
};
|
|
95
|
+
}>;
|
|
@@ -0,0 +1,163 @@
|
|
|
1
|
+
import { createHash } from "node:crypto";
|
|
2
|
+
import { CliError } from "@wirecat/cli-base";
|
|
3
|
+
import { meaningFloor } from "../embeddings/models.js";
|
|
4
|
+
import { formatMeetingReference, parseMeetingReference, } from "../meetings/index.js";
|
|
5
|
+
import { questionTerms } from "../search/question-plan.js";
|
|
6
|
+
import { refreshMeetingIndex, resolveMeetingSelection } from "./meeting-hit-search.js";
|
|
7
|
+
import { validateVector } from "./meeting-semantic.js";
|
|
8
|
+
import { RRF_K } from "./notes-search.js";
|
|
9
|
+
/** How many hits each side ranks before they are merged; a page past it has nothing more to show. */
|
|
10
|
+
const DEPTH = 200;
|
|
11
|
+
const cursorOf = (value, key) => {
|
|
12
|
+
if (value === undefined)
|
|
13
|
+
return 0;
|
|
14
|
+
let cursor;
|
|
15
|
+
try {
|
|
16
|
+
cursor = JSON.parse(Buffer.from(value, "base64url").toString("utf8"));
|
|
17
|
+
}
|
|
18
|
+
catch {
|
|
19
|
+
throw new CliError("validation_error", "the cursor is not from a combined meeting search; search again");
|
|
20
|
+
}
|
|
21
|
+
if (cursor?.version !== 1 || cursor.key !== key || !Number.isSafeInteger(cursor.offset) || cursor.offset < 0)
|
|
22
|
+
throw new CliError("validation_error", "the cursor belongs to another query, filter or model; search again");
|
|
23
|
+
return cursor.offset;
|
|
24
|
+
};
|
|
25
|
+
/**
|
|
26
|
+
* Words and meaning, ranked together by reciprocal rank fusion over one selection of meetings. A question
|
|
27
|
+
* («что решили по цене?») searches its content words; other text is the meeting query language. A word hit inside
|
|
28
|
+
* a passage the meaning search found joins it, so one discussion found both ways ranks first.
|
|
29
|
+
*/
|
|
30
|
+
export const searchMeetingsCombined = async (store, embedder, query, request) => {
|
|
31
|
+
const { maxChunks, maxRows, maxTextBytes, ...words } = request;
|
|
32
|
+
const { input, selection } = await resolveMeetingSelection(store, words);
|
|
33
|
+
const { after, limit = 20, maxReadBytes, ...filters } = input;
|
|
34
|
+
const terms = questionTerms(query);
|
|
35
|
+
const wordQuery = terms === null ? query : terms.join(" ");
|
|
36
|
+
const key = createHash("sha256")
|
|
37
|
+
.update(JSON.stringify([query, embedder.key, Object.entries(filters).filter(([name]) => name !== "signal")]))
|
|
38
|
+
.digest("base64url");
|
|
39
|
+
const offset = cursorOf(after, key);
|
|
40
|
+
let byWords = [];
|
|
41
|
+
let indexing = "ready";
|
|
42
|
+
let refused;
|
|
43
|
+
if (wordQuery.trim()) {
|
|
44
|
+
await refreshMeetingIndex(store.meetings, input.accountId, input.signal);
|
|
45
|
+
try {
|
|
46
|
+
const page = await store.meetings.searchHits(wordQuery, { ...filters, limit: DEPTH, maxReadBytes });
|
|
47
|
+
byWords = page.items;
|
|
48
|
+
indexing = page.coverage.indexing;
|
|
49
|
+
}
|
|
50
|
+
catch (error) {
|
|
51
|
+
// Text the query language refuses is still a question to ask by meaning.
|
|
52
|
+
if (!(error instanceof CliError) || error.code !== "validation_error")
|
|
53
|
+
throw error;
|
|
54
|
+
refused = error.message;
|
|
55
|
+
}
|
|
56
|
+
}
|
|
57
|
+
input.signal?.throwIfAborted();
|
|
58
|
+
const vectors = await embedder.embed([query], "query");
|
|
59
|
+
if (vectors.length !== 1)
|
|
60
|
+
throw new CliError("invalid_response", "Meeting query model returned another vector count");
|
|
61
|
+
const vector = vectors[0];
|
|
62
|
+
const ranked = await store.meetingVectors.ranked(embedder.key, validateVector(vector, embedder.dims), {
|
|
63
|
+
...filters,
|
|
64
|
+
limit: DEPTH,
|
|
65
|
+
...(maxChunks === undefined ? {} : { maxChunks }),
|
|
66
|
+
...(maxRows === undefined ? {} : { maxRows }),
|
|
67
|
+
...(maxTextBytes === undefined ? {} : { maxTextBytes }),
|
|
68
|
+
});
|
|
69
|
+
const floor = meaningFloor(embedder.key);
|
|
70
|
+
const byMeaning = ranked.items.filter(({ score }) => score > floor);
|
|
71
|
+
const merged = new Map();
|
|
72
|
+
byMeaning.forEach((hit, rank) => {
|
|
73
|
+
const reference = { type: "meeting", accountId: input.accountId, meetingId: hit.meetingId };
|
|
74
|
+
const first = formatMeetingReference({
|
|
75
|
+
...reference,
|
|
76
|
+
transcriptId: hit.transcriptId,
|
|
77
|
+
cuePosition: hit.firstPosition,
|
|
78
|
+
});
|
|
79
|
+
merged.set(`${hit.transcriptId}:${hit.firstPosition}`, {
|
|
80
|
+
ref: first,
|
|
81
|
+
meetingId: hit.meetingId,
|
|
82
|
+
scope: "transcript",
|
|
83
|
+
text: hit.text,
|
|
84
|
+
startMs: null,
|
|
85
|
+
source: { title: hit.title, startedAt: hit.startedAt, url: null },
|
|
86
|
+
by: ["meaning"],
|
|
87
|
+
score: hit.score,
|
|
88
|
+
passage: {
|
|
89
|
+
first,
|
|
90
|
+
last: formatMeetingReference({ ...reference, transcriptId: hit.transcriptId, cuePosition: hit.lastPosition }),
|
|
91
|
+
},
|
|
92
|
+
fused: 1 / (RRF_K + rank + 1),
|
|
93
|
+
});
|
|
94
|
+
});
|
|
95
|
+
byWords.forEach((hit, rank) => {
|
|
96
|
+
const share = 1 / (RRF_K + rank + 1);
|
|
97
|
+
const cue = hit.scope === "transcript" ? parseMeetingReference(hit.ref) : undefined;
|
|
98
|
+
const within = cue?.cuePosition === undefined
|
|
99
|
+
? undefined
|
|
100
|
+
: byMeaning.find((passage) => passage.transcriptId === cue.transcriptId &&
|
|
101
|
+
passage.firstPosition <= cue.cuePosition &&
|
|
102
|
+
cue.cuePosition <= passage.lastPosition);
|
|
103
|
+
const held = within && merged.get(`${within.transcriptId}:${within.firstPosition}`);
|
|
104
|
+
const found = {
|
|
105
|
+
ref: hit.ref,
|
|
106
|
+
text: hit.text,
|
|
107
|
+
startMs: hit.startMs,
|
|
108
|
+
...(hit.match === undefined ? {} : { match: hit.match }),
|
|
109
|
+
...(hit.corrections === undefined ? {} : { corrections: hit.corrections }),
|
|
110
|
+
};
|
|
111
|
+
if (held && !held.by.includes("words")) {
|
|
112
|
+
Object.assign(held, found, { by: ["words", "meaning"], fused: held.fused + share });
|
|
113
|
+
if (hit.source?.url)
|
|
114
|
+
held.source.url = hit.source.url;
|
|
115
|
+
}
|
|
116
|
+
else if (held)
|
|
117
|
+
held.fused += share;
|
|
118
|
+
else
|
|
119
|
+
merged.set(`w:${hit.ref}:${hit.scope}:${hit.id}`, {
|
|
120
|
+
...found,
|
|
121
|
+
meetingId: hit.meetingId,
|
|
122
|
+
scope: hit.scope,
|
|
123
|
+
source: {
|
|
124
|
+
title: hit.source?.title ?? null,
|
|
125
|
+
startedAt: hit.source?.startedAt ?? null,
|
|
126
|
+
url: hit.source?.url ?? null,
|
|
127
|
+
},
|
|
128
|
+
by: ["words"],
|
|
129
|
+
score: null,
|
|
130
|
+
fused: share,
|
|
131
|
+
});
|
|
132
|
+
});
|
|
133
|
+
// On a tie a line that has the words outranks a passage that only resembles the query.
|
|
134
|
+
const ordered = [...merged.values()].sort((a, b) => b.fused - a.fused ||
|
|
135
|
+
Number(b.by.includes("words")) - Number(a.by.includes("words")) ||
|
|
136
|
+
(b.score ?? -2) - (a.score ?? -2));
|
|
137
|
+
const items = ordered.slice(offset, offset + limit).map(({ fused: _, ...hit }) => hit);
|
|
138
|
+
const hasMore = ordered.length > offset + limit;
|
|
139
|
+
return {
|
|
140
|
+
items,
|
|
141
|
+
hasMore,
|
|
142
|
+
nextCursor: hasMore
|
|
143
|
+
? Buffer.from(JSON.stringify({ version: 1, key, offset: offset + limit })).toString("base64url")
|
|
144
|
+
: null,
|
|
145
|
+
selection,
|
|
146
|
+
words: {
|
|
147
|
+
query: wordQuery,
|
|
148
|
+
...(terms === null ? {} : { from: "question" }),
|
|
149
|
+
...(refused === undefined ? {} : { refused }),
|
|
150
|
+
},
|
|
151
|
+
model: { key: embedder.key, model: embedder.model, kind: embedder.kind, dims: embedder.dims },
|
|
152
|
+
coverage: {
|
|
153
|
+
archiveCompleteness: "unknown",
|
|
154
|
+
indexCompleteness: "unknown",
|
|
155
|
+
indexing,
|
|
156
|
+
/** Every vector of the selection was compared; false when `maxChunks` stopped the scan. */
|
|
157
|
+
meaningScanComplete: ranked.complete,
|
|
158
|
+
scanned: ranked.scanned,
|
|
159
|
+
/** Each side ranks this many at most; pages past them hold nothing more. */
|
|
160
|
+
depth: DEPTH,
|
|
161
|
+
},
|
|
162
|
+
};
|
|
163
|
+
};
|
|
@@ -5,6 +5,32 @@ export type { MeetingHitSearchRequest } from "../meetings/index.js";
|
|
|
5
5
|
/** Each batch commits on its own, so a search cut short by its deadline keeps the progress it made. */
|
|
6
6
|
export declare const refreshMeetingIndex: (meetings: Pick<MessageStore["meetings"], "indexSearch">, accountId: number, signal?: AbortSignal) => Promise<void>;
|
|
7
7
|
/** Name resolution changes selection only; it never assigns projects or labels from transcript words. */
|
|
8
|
+
export declare const resolveMeetingSelection: (store: {
|
|
9
|
+
knowledge?: MessageStore["knowledge"];
|
|
10
|
+
}, request: MeetingHitSearchRequest) => Promise<{
|
|
11
|
+
input: {
|
|
12
|
+
accountId: number;
|
|
13
|
+
meetingId?: number;
|
|
14
|
+
participantIdentityId?: number;
|
|
15
|
+
personId?: number;
|
|
16
|
+
projectId?: number;
|
|
17
|
+
tagId?: number;
|
|
18
|
+
eventId?: number;
|
|
19
|
+
meetingSeriesId?: number;
|
|
20
|
+
since?: number;
|
|
21
|
+
until?: number;
|
|
22
|
+
limit?: number;
|
|
23
|
+
after?: string;
|
|
24
|
+
maxReadBytes?: number;
|
|
25
|
+
signal?: AbortSignal;
|
|
26
|
+
};
|
|
27
|
+
selection: {
|
|
28
|
+
kind: VocabularyKind;
|
|
29
|
+
id: number;
|
|
30
|
+
name: string;
|
|
31
|
+
by: "explicit-link";
|
|
32
|
+
}[];
|
|
33
|
+
}>;
|
|
8
34
|
export declare const searchMeetingHits: (store: Pick<MessageStore, "meetings"> & {
|
|
9
35
|
knowledge?: MessageStore["knowledge"];
|
|
10
36
|
}, query: string, request: MeetingHitSearchRequest, options?: {
|
|
@@ -16,7 +16,7 @@ export const refreshMeetingIndex = async (meetings, accountId, signal) => {
|
|
|
16
16
|
}
|
|
17
17
|
};
|
|
18
18
|
/** Name resolution changes selection only; it never assigns projects or labels from transcript words. */
|
|
19
|
-
export const
|
|
19
|
+
export const resolveMeetingSelection = async (store, request) => {
|
|
20
20
|
const { project, tag, topic, ...input } = request;
|
|
21
21
|
input.signal?.throwIfAborted();
|
|
22
22
|
if ((project !== undefined && input.projectId !== undefined) ||
|
|
@@ -51,6 +51,10 @@ export const searchMeetingHits = async (store, query, request, options = {}) =>
|
|
|
51
51
|
input.tagId = id;
|
|
52
52
|
selection.push({ kind, id, name: entry.name, by: "explicit-link" });
|
|
53
53
|
}
|
|
54
|
+
return { input, selection };
|
|
55
|
+
};
|
|
56
|
+
export const searchMeetingHits = async (store, query, request, options = {}) => {
|
|
57
|
+
const { input, selection } = await resolveMeetingSelection(store, request);
|
|
54
58
|
if (options.refreshIndex !== false)
|
|
55
59
|
await refreshMeetingIndex(store.meetings, input.accountId, input.signal);
|
|
56
60
|
const result = await store.meetings.searchHits(query, input);
|
|
@@ -16,6 +16,7 @@ export interface MeetingEmbeddingApplyOptions {
|
|
|
16
16
|
maxVectorBytes?: number;
|
|
17
17
|
signal?: AbortSignal;
|
|
18
18
|
}
|
|
19
|
+
export declare const validateVector: (vector: Float32Array | undefined, dims: number) => Float32Array;
|
|
19
20
|
/** Embeds only this explicit query; never rebuilds chunks or generates stored vectors. */
|
|
20
21
|
export declare const readMeetingSemantic: (store: Pick<MeetingVectors, "nearest">, accountId: number, query: string, embedder: MeetingEmbedder, options?: MeetingSemanticOptions) => Promise<{
|
|
21
22
|
ranking: "scanned-candidates";
|
|
@@ -25,7 +25,7 @@ const textBudget = (options) => ({
|
|
|
25
25
|
maxRows: positive(options.maxRows ?? 10000, "transcript row budget", 100000),
|
|
26
26
|
maxTextBytes: positive(options.maxTextBytes ?? 4 * 1024 * 1024, "transcript text byte budget", 64 * 1024 * 1024),
|
|
27
27
|
});
|
|
28
|
-
const validateVector = (vector, dims) => {
|
|
28
|
+
export const validateVector = (vector, dims) => {
|
|
29
29
|
if (!(vector instanceof Float32Array) ||
|
|
30
30
|
vector.length !== dims ||
|
|
31
31
|
!vector.some((value) => value !== 0) ||
|
|
@@ -0,0 +1,13 @@
|
|
|
1
|
+
import type { MeetingHitSearchRequest, MeetingStats } from "../meetings/index.js";
|
|
2
|
+
import type { MessageStore } from "../store/store.js";
|
|
3
|
+
import { resolveMeetingSelection } from "./meeting-hit-search.js";
|
|
4
|
+
export type MeetingStatsRequest = Omit<MeetingHitSearchRequest, "limit" | "after" | "maxReadBytes"> & {
|
|
5
|
+
timeZone?: string;
|
|
6
|
+
maxRows?: number;
|
|
7
|
+
};
|
|
8
|
+
/** Talk time and lines per speaker and meetings per week over the meetings search would read, names resolved alike. */
|
|
9
|
+
export declare const readMeetingStats: (store: Pick<MessageStore, "meetings"> & {
|
|
10
|
+
knowledge?: MessageStore["knowledge"];
|
|
11
|
+
}, request: MeetingStatsRequest) => Promise<MeetingStats & {
|
|
12
|
+
selection: Awaited<ReturnType<typeof resolveMeetingSelection>>["selection"];
|
|
13
|
+
}>;
|
|
@@ -0,0 +1,12 @@
|
|
|
1
|
+
import { resolveMeetingSelection } from "./meeting-hit-search.js";
|
|
2
|
+
/** Talk time and lines per speaker and meetings per week over the meetings search would read, names resolved alike. */
|
|
3
|
+
export const readMeetingStats = async (store, request) => {
|
|
4
|
+
const { timeZone, maxRows, ...selecting } = request;
|
|
5
|
+
const { input, selection } = await resolveMeetingSelection(store, selecting);
|
|
6
|
+
const stats = await store.meetings.meetingStats({
|
|
7
|
+
...input,
|
|
8
|
+
...(timeZone === undefined ? {} : { timeZone }),
|
|
9
|
+
...(maxRows === undefined ? {} : { maxRows }),
|
|
10
|
+
});
|
|
11
|
+
return { ...stats, selection };
|
|
12
|
+
};
|
|
@@ -5,6 +5,7 @@ import { matchMeetingText, meetingQuerySql, resolveMeetingQuery } from "./meetin
|
|
|
5
5
|
import { indexedText } from "./meeting-index-values.js";
|
|
6
6
|
import { checkMeetingCancelled } from "./meeting-operation.js";
|
|
7
7
|
import { positiveReadInteger } from "./meeting-reads.js";
|
|
8
|
+
import { meetingSelectionSql } from "./meeting-selection.js";
|
|
8
9
|
import { meetingStemState } from "./meeting-stem-state.js";
|
|
9
10
|
import { read } from "./transaction.js";
|
|
10
11
|
const fail = (message) => {
|
|
@@ -112,45 +113,8 @@ export function meetingHitSearchOver({ database }) {
|
|
|
112
113
|
indexing: pending || !state.stemmer || state.through < state.watermark ? "pending" : "ready",
|
|
113
114
|
indexCompleteness: "unknown",
|
|
114
115
|
};
|
|
115
|
-
const clauses = ["m.account_id=?", "m.deleted_at IS NULL"];
|
|
116
116
|
const { sql: matching, params: queryParams } = meetingQuerySql(tree, state);
|
|
117
|
-
const params =
|
|
118
|
-
if (input.meetingId !== undefined) {
|
|
119
|
-
clauses.push("m.id=?");
|
|
120
|
-
params.push(input.meetingId);
|
|
121
|
-
}
|
|
122
|
-
if (input.eventId !== undefined) {
|
|
123
|
-
clauses.push("m.event_id=?");
|
|
124
|
-
params.push(input.eventId);
|
|
125
|
-
}
|
|
126
|
-
if (input.meetingSeriesId !== undefined) {
|
|
127
|
-
clauses.push("m.meeting_series_id=?");
|
|
128
|
-
params.push(input.meetingSeriesId);
|
|
129
|
-
}
|
|
130
|
-
if (input.since !== undefined) {
|
|
131
|
-
clauses.push("m.started_at>=?");
|
|
132
|
-
params.push(input.since);
|
|
133
|
-
}
|
|
134
|
-
if (input.until !== undefined) {
|
|
135
|
-
clauses.push("m.started_at<=?");
|
|
136
|
-
params.push(input.until);
|
|
137
|
-
}
|
|
138
|
-
if (input.participantIdentityId !== undefined) {
|
|
139
|
-
clauses.push("EXISTS (SELECT 1 FROM meeting_participants p WHERE p.meeting_id=m.id AND p.identity_id=?)");
|
|
140
|
-
params.push(input.participantIdentityId);
|
|
141
|
-
}
|
|
142
|
-
if (input.personId !== undefined) {
|
|
143
|
-
clauses.push("EXISTS (SELECT 1 FROM meeting_participants p JOIN identity_links i ON i.identity_id=p.identity_id JOIN persons person ON person.id=i.person_id WHERE p.meeting_id=m.id AND person.id=?)");
|
|
144
|
-
params.push(input.personId);
|
|
145
|
-
}
|
|
146
|
-
if (input.projectId !== undefined) {
|
|
147
|
-
clauses.push("EXISTS (SELECT 1 FROM links l JOIN projects p ON p.id=l.to_id WHERE l.from_type='meeting' AND l.from_id=m.id AND l.to_type='project' AND l.kind='member-of' AND l.anchor IS NULL AND l.confirmed=1 AND p.id=? AND p.account_id IS NULL AND p.deleted_at IS NULL)");
|
|
148
|
-
params.push(input.projectId);
|
|
149
|
-
}
|
|
150
|
-
if (input.tagId !== undefined) {
|
|
151
|
-
clauses.push("EXISTS (SELECT 1 FROM taggings g JOIN tags t ON t.id=g.tag_id WHERE g.taggable_type='meeting' AND g.taggable_id=m.id AND t.id=?)");
|
|
152
|
-
params.push(input.tagId);
|
|
153
|
-
}
|
|
117
|
+
const { clauses, params } = meetingSelectionSql(input);
|
|
154
118
|
const seek = after
|
|
155
119
|
? "WHERE h.tier>? OR (h.tier=? AND (h.started<? OR (h.started=? AND (h.meeting_id>? OR (h.meeting_id=? AND (h.kind>? OR (h.kind=? AND h.id>?)))))))"
|
|
156
120
|
: "";
|
|
@@ -0,0 +1,8 @@
|
|
|
1
|
+
import type { MeetingSearchOptions } from "../../meetings/index.js";
|
|
2
|
+
import type { SqlValue } from "../driver.js";
|
|
3
|
+
export type MeetingSelection = Pick<MeetingSearchOptions, "accountId" | "meetingId" | "eventId" | "meetingSeriesId" | "since" | "until" | "participantIdentityId" | "personId" | "projectId" | "tagId">;
|
|
4
|
+
/** The meetings a search may read, over `meetings m`: the same filters for words and for meaning. */
|
|
5
|
+
export declare const meetingSelectionSql: (input: MeetingSelection) => {
|
|
6
|
+
clauses: string[];
|
|
7
|
+
params: SqlValue[];
|
|
8
|
+
};
|
|
@@ -0,0 +1,42 @@
|
|
|
1
|
+
/** The meetings a search may read, over `meetings m`: the same filters for words and for meaning. */
|
|
2
|
+
export const meetingSelectionSql = (input) => {
|
|
3
|
+
const clauses = ["m.account_id=?", "m.deleted_at IS NULL"];
|
|
4
|
+
const params = [input.accountId];
|
|
5
|
+
if (input.meetingId !== undefined) {
|
|
6
|
+
clauses.push("m.id=?");
|
|
7
|
+
params.push(input.meetingId);
|
|
8
|
+
}
|
|
9
|
+
if (input.eventId !== undefined) {
|
|
10
|
+
clauses.push("m.event_id=?");
|
|
11
|
+
params.push(input.eventId);
|
|
12
|
+
}
|
|
13
|
+
if (input.meetingSeriesId !== undefined) {
|
|
14
|
+
clauses.push("m.meeting_series_id=?");
|
|
15
|
+
params.push(input.meetingSeriesId);
|
|
16
|
+
}
|
|
17
|
+
if (input.since !== undefined) {
|
|
18
|
+
clauses.push("m.started_at>=?");
|
|
19
|
+
params.push(input.since);
|
|
20
|
+
}
|
|
21
|
+
if (input.until !== undefined) {
|
|
22
|
+
clauses.push("m.started_at<=?");
|
|
23
|
+
params.push(input.until);
|
|
24
|
+
}
|
|
25
|
+
if (input.participantIdentityId !== undefined) {
|
|
26
|
+
clauses.push("EXISTS (SELECT 1 FROM meeting_participants p WHERE p.meeting_id=m.id AND p.identity_id=?)");
|
|
27
|
+
params.push(input.participantIdentityId);
|
|
28
|
+
}
|
|
29
|
+
if (input.personId !== undefined) {
|
|
30
|
+
clauses.push("EXISTS (SELECT 1 FROM meeting_participants p JOIN identity_links i ON i.identity_id=p.identity_id JOIN persons person ON person.id=i.person_id WHERE p.meeting_id=m.id AND person.id=?)");
|
|
31
|
+
params.push(input.personId);
|
|
32
|
+
}
|
|
33
|
+
if (input.projectId !== undefined) {
|
|
34
|
+
clauses.push("EXISTS (SELECT 1 FROM links l JOIN projects p ON p.id=l.to_id WHERE l.from_type='meeting' AND l.from_id=m.id AND l.to_type='project' AND l.kind='member-of' AND l.anchor IS NULL AND l.confirmed=1 AND p.id=? AND p.account_id IS NULL AND p.deleted_at IS NULL)");
|
|
35
|
+
params.push(input.projectId);
|
|
36
|
+
}
|
|
37
|
+
if (input.tagId !== undefined) {
|
|
38
|
+
clauses.push("EXISTS (SELECT 1 FROM taggings g JOIN tags t ON t.id=g.tag_id WHERE g.taggable_type='meeting' AND g.taggable_id=m.id AND t.id=?)");
|
|
39
|
+
params.push(input.tagId);
|
|
40
|
+
}
|
|
41
|
+
return { clauses, params };
|
|
42
|
+
};
|
|
@@ -0,0 +1,7 @@
|
|
|
1
|
+
import { type MeetingStatsStore } from "../../meetings/index.js";
|
|
2
|
+
import type { StoreContext } from "./open.js";
|
|
3
|
+
/**
|
|
4
|
+
* One transcript per meeting — its earliest current one — so a local transcript beside the provider's does not count
|
|
5
|
+
* the talk twice. Words are counted from spaces in SQL, so no transcript text leaves the database.
|
|
6
|
+
*/
|
|
7
|
+
export declare const meetingStatsOver: ({ database }: StoreContext) => MeetingStatsStore;
|
|
@@ -0,0 +1,55 @@
|
|
|
1
|
+
import { CliError } from "@wirecat/cli-base";
|
|
2
|
+
import { aggregateMeetingStats } from "../../meetings/index.js";
|
|
3
|
+
import { checkMeetingCancelled } from "./meeting-operation.js";
|
|
4
|
+
import { positiveReadInteger } from "./meeting-reads.js";
|
|
5
|
+
import { meetingSelectionSql } from "./meeting-selection.js";
|
|
6
|
+
import { read } from "./transaction.js";
|
|
7
|
+
const MOST_ROWS = 500_000;
|
|
8
|
+
/**
|
|
9
|
+
* One transcript per meeting — its earliest current one — so a local transcript beside the provider's does not count
|
|
10
|
+
* the talk twice. Words are counted from spaces in SQL, so no transcript text leaves the database.
|
|
11
|
+
*/
|
|
12
|
+
export const meetingStatsOver = ({ database }) => ({
|
|
13
|
+
async meetingStats(input) {
|
|
14
|
+
checkMeetingCancelled(input.signal);
|
|
15
|
+
positiveReadInteger(input.accountId, "accountId");
|
|
16
|
+
const maxRows = input.maxRows ?? MOST_ROWS;
|
|
17
|
+
if (!Number.isSafeInteger(maxRows) || maxRows < 1 || maxRows > MOST_ROWS)
|
|
18
|
+
throw new CliError("validation_error", `meeting stats read 1 through ${MOST_ROWS} transcript lines`);
|
|
19
|
+
return read(database, () => {
|
|
20
|
+
const { clauses, params } = meetingSelectionSql(input);
|
|
21
|
+
const sql = `WITH selected AS (SELECT m.id, m.started_at FROM meetings m WHERE ${clauses.join(" AND ")}),
|
|
22
|
+
chosen AS (SELECT min(t.id) AS id, t.meeting_id FROM meeting_transcripts t JOIN selected s ON s.id = t.meeting_id
|
|
23
|
+
WHERE t.deleted_at IS NULL AND t.superseded_at IS NULL GROUP BY t.meeting_id)
|
|
24
|
+
SELECT c.meeting_id, s.started_at, r.start_ms, r.end_ms,
|
|
25
|
+
length(trim(r.text)) - length(replace(trim(r.text), ' ', '')) + (length(trim(r.text)) > 0) AS words,
|
|
26
|
+
l.person_id, person.name AS person_name, p.identity_id, i.name AS identity_name,
|
|
27
|
+
coalesce(r.speaker_name, p.display_name) AS speaker_name
|
|
28
|
+
FROM chosen c JOIN selected s ON s.id = c.meeting_id
|
|
29
|
+
JOIN meeting_transcript_rows r ON r.meeting_transcript_id = c.id
|
|
30
|
+
LEFT JOIN meeting_participants p ON p.id = r.speaker_participant_id
|
|
31
|
+
LEFT JOIN identities i ON i.id = p.identity_id
|
|
32
|
+
LEFT JOIN identity_links l ON l.identity_id = p.identity_id
|
|
33
|
+
LEFT JOIN persons person ON person.id = l.person_id
|
|
34
|
+
LIMIT ?`;
|
|
35
|
+
const rows = database.prepare(sql).all(...params, maxRows + 1);
|
|
36
|
+
if (rows.length > maxRows)
|
|
37
|
+
throw new CliError("validation_error", `the selection has more than ${maxRows} transcript lines; narrow it`);
|
|
38
|
+
checkMeetingCancelled(input.signal);
|
|
39
|
+
const nullable = (value) => (value === null || value === undefined ? null : Number(value));
|
|
40
|
+
const text = (value) => (value === null || value === undefined ? null : String(value));
|
|
41
|
+
return aggregateMeetingStats(rows.map((row) => ({
|
|
42
|
+
meetingId: Number(row.meeting_id),
|
|
43
|
+
startedAt: nullable(row.started_at),
|
|
44
|
+
startMs: Number(row.start_ms),
|
|
45
|
+
endMs: Number(row.end_ms),
|
|
46
|
+
words: Number(row.words),
|
|
47
|
+
personId: nullable(row.person_id),
|
|
48
|
+
personName: text(row.person_name),
|
|
49
|
+
identityId: nullable(row.identity_id),
|
|
50
|
+
identityName: text(row.identity_name),
|
|
51
|
+
speakerName: text(row.speaker_name),
|
|
52
|
+
})), input.timeZone);
|
|
53
|
+
});
|
|
54
|
+
},
|
|
55
|
+
});
|
|
@@ -1,3 +1,4 @@
|
|
|
1
|
+
import { type MeetingSelection } from "./meeting-selection.js";
|
|
1
2
|
import type { StoreContext } from "./open.js";
|
|
2
3
|
interface Scope {
|
|
3
4
|
accountId: number;
|
|
@@ -61,6 +62,22 @@ export interface MeetingVectors {
|
|
|
61
62
|
chunks: number;
|
|
62
63
|
embedded: number;
|
|
63
64
|
}>;
|
|
65
|
+
/**
|
|
66
|
+
* The `limit` passages nearest to `query` among every vector of `model` in the selected meetings, up to
|
|
67
|
+
* `maxChunks` of them: one ranking over the whole selection, with text read only for the passages kept.
|
|
68
|
+
*/
|
|
69
|
+
ranked(model: string, query: Float32Array, input: MeetingSelection & Scope & TextBudget & {
|
|
70
|
+
limit: number;
|
|
71
|
+
maxChunks?: number;
|
|
72
|
+
}): Promise<{
|
|
73
|
+
items: (MeetingVectorHit & {
|
|
74
|
+
text: string;
|
|
75
|
+
title: string | null;
|
|
76
|
+
startedAt: number | null;
|
|
77
|
+
})[];
|
|
78
|
+
scanned: number;
|
|
79
|
+
complete: boolean;
|
|
80
|
+
}>;
|
|
64
81
|
}
|
|
65
82
|
export declare const meetingVectorsOver: (context: StoreContext) => MeetingVectors;
|
|
66
83
|
export {};
|
|
@@ -1,8 +1,11 @@
|
|
|
1
1
|
import { CliError } from "@wirecat/cli-base";
|
|
2
2
|
import { chunkHash, cutChunks } from "../../conversations/chunks.js";
|
|
3
3
|
import { positiveReadInteger } from "./meeting-reads.js";
|
|
4
|
+
import { meetingSelectionSql } from "./meeting-selection.js";
|
|
4
5
|
import { read, write } from "./transaction.js";
|
|
5
6
|
import { dot, saveVectors } from "./vectors.js";
|
|
7
|
+
/** Vectors read per query while ranking, so a store of any size stays within a few megabytes at a time. */
|
|
8
|
+
const RANK_BATCH = 512;
|
|
6
9
|
const LIVE = "FROM chunks k JOIN meeting_transcripts t ON k.chunkable_type = 'meeting_transcript' AND t.id = k.chunkable_id JOIN meetings m ON m.id = t.meeting_id WHERE m.account_id = ? AND m.deleted_at IS NULL AND t.deleted_at IS NULL AND t.superseded_at IS NULL";
|
|
7
10
|
const limitOf = (value, name, max = 1000) => {
|
|
8
11
|
positiveReadInteger(value, name);
|
|
@@ -233,6 +236,69 @@ export const meetingVectorsOver = (context) => ({
|
|
|
233
236
|
};
|
|
234
237
|
});
|
|
235
238
|
},
|
|
239
|
+
async ranked(model, query, input) {
|
|
240
|
+
scope(input);
|
|
241
|
+
modelOf(model);
|
|
242
|
+
const limit = limitOf(input.limit, "limit"), maxChunks = limitOf(input.maxChunks ?? 10000, "maxChunks", 100000);
|
|
243
|
+
if (!query.length || query.length > 65536 || Array.from(query).some((value) => !Number.isFinite(value)))
|
|
244
|
+
throw new CliError("validation_error", "Query vector must have finite dimensions");
|
|
245
|
+
return read(context.database, () => {
|
|
246
|
+
const { clauses, params } = meetingSelectionSql(input);
|
|
247
|
+
const batch = context.database.prepare(`SELECT k.id,k.chunkable_id,k.content_hash,k.start_offset,k.end_offset,e.dims,e.vector FROM chunks k JOIN meeting_transcripts t ON k.chunkable_type='meeting_transcript' AND t.id=k.chunkable_id JOIN meetings m ON m.id=t.meeting_id JOIN embeddings e ON e.content_hash=k.content_hash AND e.model=? WHERE ${clauses.join(" AND ")} AND t.deleted_at IS NULL AND t.superseded_at IS NULL AND k.id>? ORDER BY k.id LIMIT ?`);
|
|
248
|
+
const best = [];
|
|
249
|
+
let scanned = 0;
|
|
250
|
+
let after = 0;
|
|
251
|
+
let complete = true;
|
|
252
|
+
for (;;) {
|
|
253
|
+
input.signal?.throwIfAborted();
|
|
254
|
+
const take = Math.min(RANK_BATCH, maxChunks - scanned);
|
|
255
|
+
if (take <= 0) {
|
|
256
|
+
complete = !batch.get(model, ...params, after, 1);
|
|
257
|
+
break;
|
|
258
|
+
}
|
|
259
|
+
const rows = batch.all(model, ...params, after, take);
|
|
260
|
+
for (const row of rows) {
|
|
261
|
+
if (Number(row.dims) !== query.length ||
|
|
262
|
+
!(row.vector instanceof Uint8Array) ||
|
|
263
|
+
row.vector.byteLength !== query.length * 4)
|
|
264
|
+
throw new CliError("validation_error", "Stored vector dimensions do not match query");
|
|
265
|
+
const score = dot(query, row.vector);
|
|
266
|
+
if (!Number.isFinite(score))
|
|
267
|
+
throw new CliError("invalid_response", "Stored vector contains non-finite values");
|
|
268
|
+
if (best.length < limit || score > (best.at(-1)?.score ?? Number.NEGATIVE_INFINITY)) {
|
|
269
|
+
const { vector: _, ...kept } = row;
|
|
270
|
+
const at = best.findIndex((held) => held.score < score);
|
|
271
|
+
best.splice(at < 0 ? best.length : at, 0, { row: kept, score });
|
|
272
|
+
if (best.length > limit)
|
|
273
|
+
best.pop();
|
|
274
|
+
}
|
|
275
|
+
}
|
|
276
|
+
scanned += rows.length;
|
|
277
|
+
after = Number(rows.at(-1)?.id ?? after);
|
|
278
|
+
if (rows.length < take)
|
|
279
|
+
break;
|
|
280
|
+
}
|
|
281
|
+
const load = textLoader(context, input);
|
|
282
|
+
const meeting = context.database.prepare("SELECT title,started_at FROM meetings WHERE id=?");
|
|
283
|
+
const items = best.map(({ row, score }) => {
|
|
284
|
+
input.signal?.throwIfAborted();
|
|
285
|
+
const text = chunkText(context, input.accountId, row, input, input.signal, load(Number(row.chunkable_id)));
|
|
286
|
+
const about = meeting.get(text.meetingId);
|
|
287
|
+
return {
|
|
288
|
+
title: about?.title == null ? null : String(about.title),
|
|
289
|
+
startedAt: about?.started_at == null ? null : Number(about.started_at),
|
|
290
|
+
meetingId: text.meetingId,
|
|
291
|
+
transcriptId: Number(row.chunkable_id),
|
|
292
|
+
firstPosition: text.firstPosition,
|
|
293
|
+
lastPosition: text.lastPosition,
|
|
294
|
+
hash: String(row.content_hash),
|
|
295
|
+
score,
|
|
296
|
+
text: text.text,
|
|
297
|
+
};
|
|
298
|
+
});
|
|
299
|
+
return { items, scanned, complete };
|
|
300
|
+
});
|
|
301
|
+
},
|
|
236
302
|
async status(input) {
|
|
237
303
|
scope(input);
|
|
238
304
|
modelOf(input.model);
|
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
import type { Attachment, MeetingContextStore, MeetingParticipantSearchStore, MeetingPreparationContextStore, MeetingPreparationKnowledgeStore, MeetingPreparationNotesStore, MeetingRelatedHistoryStore, MeetingSearchIndexStore, MeetingSearchStore, MeetingSeriesHistoryStore, MeetingStore, MeetingTranscriptReceiptStore, MeetingTranscriptStore, NewRecord } from "../../meetings/index.js";
|
|
1
|
+
import type { Attachment, MeetingContextStore, MeetingParticipantSearchStore, MeetingPreparationContextStore, MeetingPreparationKnowledgeStore, MeetingPreparationNotesStore, MeetingRelatedHistoryStore, MeetingSearchIndexStore, MeetingSearchStore, MeetingSeriesHistoryStore, MeetingStatsStore, MeetingStore, MeetingTranscriptReceiptStore, MeetingTranscriptStore, NewRecord } from "../../meetings/index.js";
|
|
2
2
|
import type { CacheDatabase } from "../driver.js";
|
|
3
3
|
import { type MeetingContextReadCapabilities, type MeetingReadCapabilities } from "./meeting-reads.js";
|
|
4
4
|
import type { StoreContext } from "./open.js";
|
|
@@ -11,7 +11,7 @@ export declare const saveAttachment: (database: CacheDatabase, type: string, par
|
|
|
11
11
|
export declare const drainMeetingIndex: (database: CacheDatabase, batch?: number) => number;
|
|
12
12
|
/** Each word of the query, as a prefix, all required. */
|
|
13
13
|
export declare const wordsQuery: (query: string) => string | null;
|
|
14
|
-
export type StoredMeetings = MeetingStore & MeetingPreparationContextStore & MeetingPreparationKnowledgeStore & MeetingPreparationNotesStore & MeetingRelatedHistoryStore & MeetingSeriesHistoryStore & MeetingTranscriptStore & MeetingReadCapabilities & MeetingContextReadCapabilities & MeetingParticipantSearchStore & MeetingSearchStore & MeetingTranscriptReceiptStore & MeetingContextStore & MeetingSearchIndexStore;
|
|
14
|
+
export type StoredMeetings = MeetingStore & MeetingPreparationContextStore & MeetingPreparationKnowledgeStore & MeetingPreparationNotesStore & MeetingRelatedHistoryStore & MeetingSeriesHistoryStore & MeetingTranscriptStore & MeetingReadCapabilities & MeetingContextReadCapabilities & MeetingParticipantSearchStore & MeetingSearchStore & MeetingTranscriptReceiptStore & MeetingContextStore & MeetingSearchIndexStore & MeetingStatsStore;
|
|
15
15
|
/** The `MeetingStore` of `src/meetings/` over the shared store. */
|
|
16
16
|
export declare const meetingStoreOver: (context: StoreContext) => StoredMeetings;
|
|
17
17
|
export {};
|
|
@@ -16,6 +16,7 @@ import { meetingPreparationNotesOver } from "./meeting-preparation-notes.js";
|
|
|
16
16
|
import { meetingReadsOver } from "./meeting-reads.js";
|
|
17
17
|
import { meetingRelatedHistoryOver } from "./meeting-related-history.js";
|
|
18
18
|
import { meetingSeriesHistoryOver } from "./meeting-series-history.js";
|
|
19
|
+
import { meetingStatsOver } from "./meeting-stats.js";
|
|
19
20
|
import { invalidateMeetingStems } from "./meeting-stem-state.js";
|
|
20
21
|
import { meetingOf, transcriptOf, transcriptRowOf } from "./meeting-values.js";
|
|
21
22
|
import { queryFilter } from "./query-filter.js";
|
|
@@ -571,6 +572,7 @@ export const meetingStoreOver = (context) => {
|
|
|
571
572
|
...meetingPreparationNotesOver(context),
|
|
572
573
|
...meetingRelatedHistoryOver(context),
|
|
573
574
|
...meetingHitSearchOver(context),
|
|
575
|
+
...meetingStatsOver(context),
|
|
574
576
|
...meetingParticipantSearchOver(context),
|
|
575
577
|
...meetingHitIndexOver(context),
|
|
576
578
|
...meetingContextLinksOver(context),
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@wirecat/cli-core",
|
|
3
|
-
"version": "0.
|
|
3
|
+
"version": "0.389.0",
|
|
4
4
|
"description": "The messenger-neutral half of a messaging CLI: domain model, message locators, rendering, name resolution, the SQLite seam, the command skeleton and shared read commands, the send guard, run records and the message store",
|
|
5
5
|
"license": "Apache-2.0",
|
|
6
6
|
"author": "Viacheslav Ptsarev",
|