@oh-my-pi/omp-stats 14.9.5 → 14.9.7
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/package.json +3 -3
- package/src/aggregator.ts +146 -36
- package/src/client/components/BehaviorChart.tsx +11 -4
- package/src/client/components/BehaviorModelsTable.tsx +62 -19
- package/src/client/components/BehaviorSummary.tsx +30 -10
- package/src/client/types.ts +15 -6
- package/src/db.ts +151 -38
- package/src/index.ts +29 -3
- package/src/parser.ts +31 -14
- package/src/sync-worker.ts +31 -0
- package/src/types.ts +42 -10
- package/src/user-metrics.ts +217 -17
package/src/db.ts
CHANGED
|
@@ -14,6 +14,7 @@ import type {
|
|
|
14
14
|
ModelStats,
|
|
15
15
|
ModelTimeSeriesPoint,
|
|
16
16
|
TimeSeriesPoint,
|
|
17
|
+
UserMessageLink,
|
|
17
18
|
UserMessageStats,
|
|
18
19
|
} from "./types";
|
|
19
20
|
|
|
@@ -98,9 +99,12 @@ export async function initDb(): Promise<Database> {
|
|
|
98
99
|
provider TEXT,
|
|
99
100
|
chars INTEGER NOT NULL,
|
|
100
101
|
words INTEGER NOT NULL,
|
|
101
|
-
|
|
102
|
+
yelling INTEGER NOT NULL,
|
|
102
103
|
profanity INTEGER NOT NULL,
|
|
103
|
-
|
|
104
|
+
anguish INTEGER NOT NULL,
|
|
105
|
+
negation INTEGER NOT NULL DEFAULT 0,
|
|
106
|
+
repetition INTEGER NOT NULL DEFAULT 0,
|
|
107
|
+
blame INTEGER NOT NULL DEFAULT 0,
|
|
104
108
|
UNIQUE(session_file, entry_id)
|
|
105
109
|
);
|
|
106
110
|
|
|
@@ -118,16 +122,30 @@ export async function initDb(): Promise<Database> {
|
|
|
118
122
|
db.exec("ALTER TABLE messages ADD COLUMN premium_requests REAL NOT NULL DEFAULT 0");
|
|
119
123
|
}
|
|
120
124
|
db.exec("UPDATE messages SET premium_requests = 0 WHERE premium_requests IS NULL");
|
|
121
|
-
//
|
|
122
|
-
//
|
|
123
|
-
//
|
|
124
|
-
//
|
|
125
|
-
//
|
|
126
|
-
//
|
|
125
|
+
// Each behavior-metric bump invalidates previously-ingested rows. We detect
|
|
126
|
+
// the stale schema by column name and drop the table; `IF NOT EXISTS` above
|
|
127
|
+
// already produced the new schema, but we want a clean wipe + re-ingest.
|
|
128
|
+
// `backfillUserMessages` then clears `file_offsets` so the next sync
|
|
129
|
+
// re-parses every session under the current metric definitions.
|
|
130
|
+
// v1 -> v2: yelling sentences replace `caps_words`.
|
|
131
|
+
// v2 -> v3: `drama_runs` folded into a single `anguish` signal that
|
|
132
|
+
// also captures elongated interjections, `dude`, and dot runs,
|
|
133
|
+
// gated on a stripped prose-line budget.
|
|
134
|
+
// v3 -> v4: added `negation`, `repetition`, `blame` frustration signals
|
|
135
|
+
// plus profanity dictionary expansion + word-boundary fix.
|
|
136
|
+
// v4 -> v5: column `yelling_sentences` renamed to `yelling` to match
|
|
137
|
+
// the other single-word signal columns.
|
|
127
138
|
const userMessageColumns = db.prepare("PRAGMA table_info(user_messages)").all() as {
|
|
128
139
|
name: string;
|
|
129
140
|
}[];
|
|
130
|
-
|
|
141
|
+
const hasStaleColumn =
|
|
142
|
+
userMessageColumns.length > 0 &&
|
|
143
|
+
(userMessageColumns.some(column => column.name === "caps_words") ||
|
|
144
|
+
userMessageColumns.some(column => column.name === "drama_runs") ||
|
|
145
|
+
userMessageColumns.some(column => column.name === "yelling_sentences"));
|
|
146
|
+
const hasV4Columns = userMessageColumns.some(column => column.name === "negation");
|
|
147
|
+
const hasOldUserMessages = userMessageColumns.length > 0;
|
|
148
|
+
if (hasStaleColumn || (hasOldUserMessages && !hasV4Columns)) {
|
|
131
149
|
db.exec("DROP TABLE user_messages");
|
|
132
150
|
db.exec(`
|
|
133
151
|
CREATE TABLE user_messages (
|
|
@@ -140,9 +158,12 @@ export async function initDb(): Promise<Database> {
|
|
|
140
158
|
provider TEXT,
|
|
141
159
|
chars INTEGER NOT NULL,
|
|
142
160
|
words INTEGER NOT NULL,
|
|
143
|
-
|
|
161
|
+
yelling INTEGER NOT NULL,
|
|
144
162
|
profanity INTEGER NOT NULL,
|
|
145
|
-
|
|
163
|
+
anguish INTEGER NOT NULL,
|
|
164
|
+
negation INTEGER NOT NULL DEFAULT 0,
|
|
165
|
+
repetition INTEGER NOT NULL DEFAULT 0,
|
|
166
|
+
blame INTEGER NOT NULL DEFAULT 0,
|
|
146
167
|
UNIQUE(session_file, entry_id)
|
|
147
168
|
);
|
|
148
169
|
CREATE INDEX IF NOT EXISTS idx_user_messages_timestamp ON user_messages(timestamp);
|
|
@@ -150,6 +171,7 @@ export async function initDb(): Promise<Database> {
|
|
|
150
171
|
`);
|
|
151
172
|
}
|
|
152
173
|
backfillUserMessages(db);
|
|
174
|
+
repairUserMessageLinks(db);
|
|
153
175
|
backfillMissingCatalogCosts(db);
|
|
154
176
|
return db;
|
|
155
177
|
}
|
|
@@ -704,11 +726,19 @@ export function getCostTimeSeries(days = 90, cutoff?: number | null): CostTimeSe
|
|
|
704
726
|
* - v1: initial introduction of `user_messages`.
|
|
705
727
|
* - v2: yelling-sentence metric replaces caps-word counts; existing rows are
|
|
706
728
|
* computed under the old definition and must be discarded.
|
|
729
|
+
* - v3: drama runs collapsed into `anguish` (drama + elongated interjections
|
|
730
|
+
* + `dude` + dot runs), scored on a stripped prose body and gated on
|
|
731
|
+
* line count. Existing rows used the narrower definition.
|
|
732
|
+
* - v4: added `negation` / `repetition` / `blame` signals and fixed a
|
|
733
|
+
* latent word-boundary bug in the profanity / anguish regexes that had
|
|
734
|
+
* left those metrics matching nothing in real prose.
|
|
735
|
+
* - v5: renamed `yelling_sentences` column to `yelling` to match the other
|
|
736
|
+
* single-word signal columns (profanity, anguish, negation, ...).
|
|
707
737
|
*
|
|
708
|
-
* Existing `messages` rows are unaffected
|
|
738
|
+
* Existing `messages` rows are unaffected - `INSERT OR IGNORE` keeps them.
|
|
709
739
|
*/
|
|
710
740
|
function backfillUserMessages(database: Database): void {
|
|
711
|
-
const row = database.prepare("SELECT value FROM meta WHERE key = '
|
|
741
|
+
const row = database.prepare("SELECT value FROM meta WHERE key = 'user_messages_v5'").get() as
|
|
712
742
|
| { value: string }
|
|
713
743
|
| undefined;
|
|
714
744
|
if (row) return;
|
|
@@ -717,7 +747,27 @@ function backfillUserMessages(database: Database): void {
|
|
|
717
747
|
database.exec("DELETE FROM file_offsets");
|
|
718
748
|
database
|
|
719
749
|
.prepare("INSERT OR REPLACE INTO meta (key, value) VALUES (?, ?)")
|
|
720
|
-
.run("
|
|
750
|
+
.run("user_messages_v5", String(Date.now()));
|
|
751
|
+
}
|
|
752
|
+
|
|
753
|
+
/**
|
|
754
|
+
* One-shot wipe of `file_offsets` to force `parseSessionFile` to re-parse
|
|
755
|
+
* every session from byte zero. We don't touch `user_messages`; the parser
|
|
756
|
+
* now emits a `UserMessageLink` for every assistant->parent pair, and the
|
|
757
|
+
* guarded `updateUserMessageLinks` UPDATE fixes any row whose `model` was
|
|
758
|
+
* left NULL by the old in-pass-only linking logic. Idempotent: gated by a
|
|
759
|
+
* sentinel row in `meta`.
|
|
760
|
+
*/
|
|
761
|
+
function repairUserMessageLinks(database: Database): void {
|
|
762
|
+
const row = database.prepare("SELECT value FROM meta WHERE key = 'user_message_links_v1'").get() as
|
|
763
|
+
| { value: string }
|
|
764
|
+
| undefined;
|
|
765
|
+
if (row) return;
|
|
766
|
+
|
|
767
|
+
database.exec("DELETE FROM file_offsets");
|
|
768
|
+
database
|
|
769
|
+
.prepare("INSERT OR REPLACE INTO meta (key, value) VALUES (?, ?)")
|
|
770
|
+
.run("user_message_links_v1", String(Date.now()));
|
|
721
771
|
}
|
|
722
772
|
|
|
723
773
|
/**
|
|
@@ -729,8 +779,9 @@ export function insertUserMessageStats(stats: UserMessageStats[]): number {
|
|
|
729
779
|
const stmt = db.prepare(`
|
|
730
780
|
INSERT OR IGNORE INTO user_messages (
|
|
731
781
|
session_file, entry_id, folder, timestamp, model, provider,
|
|
732
|
-
chars, words,
|
|
733
|
-
|
|
782
|
+
chars, words, yelling, profanity, anguish,
|
|
783
|
+
negation, repetition, blame
|
|
784
|
+
) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
|
|
734
785
|
`);
|
|
735
786
|
|
|
736
787
|
let inserted = 0;
|
|
@@ -745,9 +796,12 @@ export function insertUserMessageStats(stats: UserMessageStats[]): number {
|
|
|
745
796
|
s.provider,
|
|
746
797
|
s.chars,
|
|
747
798
|
s.words,
|
|
748
|
-
s.
|
|
799
|
+
s.yelling,
|
|
749
800
|
s.profanity,
|
|
750
|
-
s.
|
|
801
|
+
s.anguish,
|
|
802
|
+
s.negation,
|
|
803
|
+
s.repetition,
|
|
804
|
+
s.blame,
|
|
751
805
|
);
|
|
752
806
|
if (result.changes > 0) inserted++;
|
|
753
807
|
}
|
|
@@ -756,6 +810,35 @@ export function insertUserMessageStats(stats: UserMessageStats[]): number {
|
|
|
756
810
|
return inserted;
|
|
757
811
|
}
|
|
758
812
|
|
|
813
|
+
/**
|
|
814
|
+
* Backfill the responding `model`/`provider` on user-message rows that were
|
|
815
|
+
* persisted before their assistant reply was parsed (a side effect of
|
|
816
|
+
* incremental `fromOffset` syncing: the `userByEntryId` map in
|
|
817
|
+
* `parseSessionFile` only spans a single pass). Each row is updated at most
|
|
818
|
+
* once because the `model IS NULL` guard short-circuits subsequent passes.
|
|
819
|
+
*
|
|
820
|
+
* Returns the number of rows actually updated.
|
|
821
|
+
*/
|
|
822
|
+
export function updateUserMessageLinks(links: UserMessageLink[]): number {
|
|
823
|
+
if (!db || links.length === 0) return 0;
|
|
824
|
+
|
|
825
|
+
const stmt = db.prepare(`
|
|
826
|
+
UPDATE user_messages
|
|
827
|
+
SET model = ?, provider = ?
|
|
828
|
+
WHERE session_file = ? AND entry_id = ? AND model IS NULL
|
|
829
|
+
`);
|
|
830
|
+
|
|
831
|
+
let updated = 0;
|
|
832
|
+
const apply = db.transaction(() => {
|
|
833
|
+
for (const link of links) {
|
|
834
|
+
const result = stmt.run(link.model, link.provider, link.sessionFile, link.entryId);
|
|
835
|
+
if (result.changes > 0) updated++;
|
|
836
|
+
}
|
|
837
|
+
});
|
|
838
|
+
apply();
|
|
839
|
+
return updated;
|
|
840
|
+
}
|
|
841
|
+
|
|
759
842
|
const UNKNOWN_MODEL = "unknown";
|
|
760
843
|
|
|
761
844
|
interface BehaviorSeriesRow {
|
|
@@ -763,9 +846,12 @@ interface BehaviorSeriesRow {
|
|
|
763
846
|
model: string;
|
|
764
847
|
provider: string;
|
|
765
848
|
messages: number;
|
|
766
|
-
|
|
849
|
+
yelling: number | null;
|
|
767
850
|
profanity: number | null;
|
|
768
|
-
|
|
851
|
+
anguish: number | null;
|
|
852
|
+
negation: number | null;
|
|
853
|
+
repetition: number | null;
|
|
854
|
+
blame: number | null;
|
|
769
855
|
chars: number | null;
|
|
770
856
|
}
|
|
771
857
|
|
|
@@ -781,9 +867,12 @@ export function getBehaviorTimeSeries(cutoff?: number | null): BehaviorTimeSerie
|
|
|
781
867
|
COALESCE(model, ?) as model,
|
|
782
868
|
COALESCE(provider, ?) as provider,
|
|
783
869
|
COUNT(*) as messages,
|
|
784
|
-
SUM(
|
|
870
|
+
SUM(yelling) as yelling,
|
|
785
871
|
SUM(profanity) as profanity,
|
|
786
|
-
SUM(
|
|
872
|
+
SUM(anguish) as anguish,
|
|
873
|
+
SUM(negation) as negation,
|
|
874
|
+
SUM(repetition) as repetition,
|
|
875
|
+
SUM(blame) as blame,
|
|
787
876
|
SUM(chars) as chars
|
|
788
877
|
FROM user_messages
|
|
789
878
|
${hasCutoff ? "WHERE timestamp >= ?" : ""}
|
|
@@ -798,18 +887,24 @@ export function getBehaviorTimeSeries(cutoff?: number | null): BehaviorTimeSerie
|
|
|
798
887
|
model: row.model,
|
|
799
888
|
provider: row.provider,
|
|
800
889
|
messages: row.messages,
|
|
801
|
-
|
|
890
|
+
yelling: row.yelling ?? 0,
|
|
802
891
|
profanity: row.profanity ?? 0,
|
|
803
|
-
|
|
892
|
+
anguish: row.anguish ?? 0,
|
|
893
|
+
negation: row.negation ?? 0,
|
|
894
|
+
repetition: row.repetition ?? 0,
|
|
895
|
+
blame: row.blame ?? 0,
|
|
804
896
|
chars: row.chars ?? 0,
|
|
805
897
|
}));
|
|
806
898
|
}
|
|
807
899
|
|
|
808
900
|
interface BehaviorOverallRow {
|
|
809
901
|
total_messages: number;
|
|
810
|
-
|
|
902
|
+
total_yelling: number | null;
|
|
811
903
|
total_profanity: number | null;
|
|
812
|
-
|
|
904
|
+
total_anguish: number | null;
|
|
905
|
+
total_negation: number | null;
|
|
906
|
+
total_repetition: number | null;
|
|
907
|
+
total_blame: number | null;
|
|
813
908
|
total_chars: number | null;
|
|
814
909
|
first_timestamp: number | null;
|
|
815
910
|
last_timestamp: number | null;
|
|
@@ -821,9 +916,12 @@ interface BehaviorOverallRow {
|
|
|
821
916
|
export function getBehaviorOverall(cutoff?: number | null): BehaviorOverallStats {
|
|
822
917
|
const empty: BehaviorOverallStats = {
|
|
823
918
|
totalMessages: 0,
|
|
824
|
-
|
|
919
|
+
totalYelling: 0,
|
|
825
920
|
totalProfanity: 0,
|
|
826
|
-
|
|
921
|
+
totalAnguish: 0,
|
|
922
|
+
totalNegation: 0,
|
|
923
|
+
totalRepetition: 0,
|
|
924
|
+
totalBlame: 0,
|
|
827
925
|
totalChars: 0,
|
|
828
926
|
firstTimestamp: 0,
|
|
829
927
|
lastTimestamp: 0,
|
|
@@ -833,9 +931,12 @@ export function getBehaviorOverall(cutoff?: number | null): BehaviorOverallStats
|
|
|
833
931
|
const stmt = db.prepare(`
|
|
834
932
|
SELECT
|
|
835
933
|
COUNT(*) as total_messages,
|
|
836
|
-
SUM(
|
|
934
|
+
SUM(yelling) as total_yelling,
|
|
837
935
|
SUM(profanity) as total_profanity,
|
|
838
|
-
SUM(
|
|
936
|
+
SUM(anguish) as total_anguish,
|
|
937
|
+
SUM(negation) as total_negation,
|
|
938
|
+
SUM(repetition) as total_repetition,
|
|
939
|
+
SUM(blame) as total_blame,
|
|
839
940
|
SUM(chars) as total_chars,
|
|
840
941
|
MIN(timestamp) as first_timestamp,
|
|
841
942
|
MAX(timestamp) as last_timestamp
|
|
@@ -846,9 +947,12 @@ export function getBehaviorOverall(cutoff?: number | null): BehaviorOverallStats
|
|
|
846
947
|
if (!row?.total_messages) return empty;
|
|
847
948
|
return {
|
|
848
949
|
totalMessages: row.total_messages,
|
|
849
|
-
|
|
950
|
+
totalYelling: row.total_yelling ?? 0,
|
|
850
951
|
totalProfanity: row.total_profanity ?? 0,
|
|
851
|
-
|
|
952
|
+
totalAnguish: row.total_anguish ?? 0,
|
|
953
|
+
totalNegation: row.total_negation ?? 0,
|
|
954
|
+
totalRepetition: row.total_repetition ?? 0,
|
|
955
|
+
totalBlame: row.total_blame ?? 0,
|
|
852
956
|
totalChars: row.total_chars ?? 0,
|
|
853
957
|
firstTimestamp: row.first_timestamp ?? 0,
|
|
854
958
|
lastTimestamp: row.last_timestamp ?? 0,
|
|
@@ -859,9 +963,12 @@ interface BehaviorByModelRow {
|
|
|
859
963
|
model: string;
|
|
860
964
|
provider: string;
|
|
861
965
|
total_messages: number;
|
|
862
|
-
|
|
966
|
+
total_yelling: number | null;
|
|
863
967
|
total_profanity: number | null;
|
|
864
|
-
|
|
968
|
+
total_anguish: number | null;
|
|
969
|
+
total_negation: number | null;
|
|
970
|
+
total_repetition: number | null;
|
|
971
|
+
total_blame: number | null;
|
|
865
972
|
total_chars: number | null;
|
|
866
973
|
last_timestamp: number | null;
|
|
867
974
|
}
|
|
@@ -878,9 +985,12 @@ export function getBehaviorByModel(cutoff?: number | null): BehaviorModelStats[]
|
|
|
878
985
|
COALESCE(model, ?) as model,
|
|
879
986
|
COALESCE(provider, ?) as provider,
|
|
880
987
|
COUNT(*) as total_messages,
|
|
881
|
-
SUM(
|
|
988
|
+
SUM(yelling) as total_yelling,
|
|
882
989
|
SUM(profanity) as total_profanity,
|
|
883
|
-
SUM(
|
|
990
|
+
SUM(anguish) as total_anguish,
|
|
991
|
+
SUM(negation) as total_negation,
|
|
992
|
+
SUM(repetition) as total_repetition,
|
|
993
|
+
SUM(blame) as total_blame,
|
|
884
994
|
SUM(chars) as total_chars,
|
|
885
995
|
MAX(timestamp) as last_timestamp
|
|
886
996
|
FROM user_messages
|
|
@@ -895,9 +1005,12 @@ export function getBehaviorByModel(cutoff?: number | null): BehaviorModelStats[]
|
|
|
895
1005
|
model: row.model,
|
|
896
1006
|
provider: row.provider,
|
|
897
1007
|
totalMessages: row.total_messages,
|
|
898
|
-
|
|
1008
|
+
totalYelling: row.total_yelling ?? 0,
|
|
899
1009
|
totalProfanity: row.total_profanity ?? 0,
|
|
900
|
-
|
|
1010
|
+
totalAnguish: row.total_anguish ?? 0,
|
|
1011
|
+
totalNegation: row.total_negation ?? 0,
|
|
1012
|
+
totalRepetition: row.total_repetition ?? 0,
|
|
1013
|
+
totalBlame: row.total_blame ?? 0,
|
|
901
1014
|
totalChars: row.total_chars ?? 0,
|
|
902
1015
|
lastTimestamp: row.last_timestamp ?? 0,
|
|
903
1016
|
}));
|
package/src/index.ts
CHANGED
|
@@ -6,7 +6,13 @@ import { getDashboardStats, getTotalMessageCount, syncAllSessions } from "./aggr
|
|
|
6
6
|
import { closeDb } from "./db";
|
|
7
7
|
import { startServer } from "./server";
|
|
8
8
|
|
|
9
|
-
export {
|
|
9
|
+
export {
|
|
10
|
+
getDashboardStats,
|
|
11
|
+
getTotalMessageCount,
|
|
12
|
+
type SyncOptions,
|
|
13
|
+
type SyncProgress,
|
|
14
|
+
syncAllSessions,
|
|
15
|
+
} from "./aggregator";
|
|
10
16
|
export { closeDb } from "./db";
|
|
11
17
|
export { startServer } from "./server";
|
|
12
18
|
export type {
|
|
@@ -112,8 +118,28 @@ Examples:
|
|
|
112
118
|
|
|
113
119
|
try {
|
|
114
120
|
// Sync first
|
|
115
|
-
|
|
116
|
-
|
|
121
|
+
const tty = process.stderr.isTTY === true;
|
|
122
|
+
process.stderr.write("Syncing session files...\n");
|
|
123
|
+
let lastWidth = 0;
|
|
124
|
+
let lastRender = 0;
|
|
125
|
+
const { processed, files } = await syncAllSessions({
|
|
126
|
+
onProgress: event => {
|
|
127
|
+
if (!tty) return;
|
|
128
|
+
const now = Date.now();
|
|
129
|
+
if (event.current < event.total && now - lastRender < 33) return;
|
|
130
|
+
lastRender = now;
|
|
131
|
+
const marker = "/sessions/";
|
|
132
|
+
const idx = event.sessionFile.indexOf(marker);
|
|
133
|
+
const short = idx >= 0 ? event.sessionFile.slice(idx + marker.length) : event.sessionFile;
|
|
134
|
+
const pct = ((event.current / event.total) * 100).toFixed(0).padStart(3, " ");
|
|
135
|
+
const line = `[${event.current}/${event.total}] ${pct}% ${short}`;
|
|
136
|
+
const columns = process.stderr.columns ?? 120;
|
|
137
|
+
const clipped = line.length > columns - 1 ? `${line.slice(0, columns - 2)}\u2026` : line;
|
|
138
|
+
process.stderr.write(`\r${clipped.padEnd(lastWidth)}`);
|
|
139
|
+
lastWidth = clipped.length;
|
|
140
|
+
},
|
|
141
|
+
});
|
|
142
|
+
if (tty && lastWidth > 0) process.stderr.write(`\r${" ".repeat(lastWidth)}\r`);
|
|
117
143
|
const total = await getTotalMessageCount();
|
|
118
144
|
console.log(`Synced ${processed} new entries from ${files} files (${total} total)\n`);
|
|
119
145
|
|
package/src/parser.ts
CHANGED
|
@@ -2,7 +2,7 @@ import * as fs from "node:fs/promises";
|
|
|
2
2
|
import * as path from "node:path";
|
|
3
3
|
import type { AssistantMessage } from "@oh-my-pi/pi-ai";
|
|
4
4
|
import { getSessionsDir, isEnoent } from "@oh-my-pi/pi-utils";
|
|
5
|
-
import type { MessageStats, SessionEntry, SessionMessageEntry, UserMessageStats } from "./types";
|
|
5
|
+
import type { MessageStats, SessionEntry, SessionMessageEntry, UserMessageLink, UserMessageStats } from "./types";
|
|
6
6
|
import { computeUserMessageMetrics } from "./user-metrics";
|
|
7
7
|
|
|
8
8
|
/**
|
|
@@ -71,9 +71,12 @@ function extractUserStats(sessionFile: string, folder: string, entry: SessionMes
|
|
|
71
71
|
provider: null,
|
|
72
72
|
chars: metrics.chars,
|
|
73
73
|
words: metrics.words,
|
|
74
|
-
|
|
74
|
+
yelling: metrics.yelling,
|
|
75
75
|
profanity: metrics.profanity,
|
|
76
|
-
|
|
76
|
+
anguish: metrics.anguish,
|
|
77
|
+
negation: metrics.negation,
|
|
78
|
+
repetition: metrics.repetition,
|
|
79
|
+
blame: metrics.blame,
|
|
77
80
|
};
|
|
78
81
|
}
|
|
79
82
|
|
|
@@ -131,21 +134,26 @@ function parseSessionEntriesLenient(bytes: Uint8Array): { entries: SessionEntry[
|
|
|
131
134
|
* Parse a session file and extract all assistant message stats.
|
|
132
135
|
* Uses incremental reading with offset tracking.
|
|
133
136
|
*/
|
|
134
|
-
export
|
|
135
|
-
|
|
136
|
-
|
|
137
|
-
|
|
137
|
+
export interface ParseSessionResult {
|
|
138
|
+
stats: MessageStats[];
|
|
139
|
+
userStats: UserMessageStats[];
|
|
140
|
+
userLinks: UserMessageLink[];
|
|
141
|
+
newOffset: number;
|
|
142
|
+
}
|
|
143
|
+
|
|
144
|
+
export async function parseSessionFile(sessionPath: string, fromOffset = 0): Promise<ParseSessionResult> {
|
|
138
145
|
let bytes: Uint8Array;
|
|
139
146
|
try {
|
|
140
147
|
bytes = await Bun.file(sessionPath).bytes();
|
|
141
148
|
} catch (err) {
|
|
142
|
-
if (isEnoent(err)) return { stats: [], userStats: [], newOffset: fromOffset };
|
|
149
|
+
if (isEnoent(err)) return { stats: [], userStats: [], userLinks: [], newOffset: fromOffset };
|
|
143
150
|
throw err;
|
|
144
151
|
}
|
|
145
152
|
|
|
146
153
|
const folder = extractFolderFromPath(sessionPath);
|
|
147
154
|
const stats: MessageStats[] = [];
|
|
148
155
|
const userStats: UserMessageStats[] = [];
|
|
156
|
+
const userLinks: UserMessageLink[] = [];
|
|
149
157
|
const userByEntryId = new Map<string, UserMessageStats>();
|
|
150
158
|
const start = Math.max(0, Math.min(fromOffset, bytes.length));
|
|
151
159
|
const unprocessed = bytes.subarray(start);
|
|
@@ -165,17 +173,26 @@ export async function parseSessionFile(
|
|
|
165
173
|
// Link assistant's responding model back to the user message it answered.
|
|
166
174
|
const parentId = (entry as SessionMessageEntry).parentId;
|
|
167
175
|
if (parentId) {
|
|
168
|
-
const
|
|
169
|
-
if (
|
|
170
|
-
|
|
171
|
-
|
|
172
|
-
|
|
176
|
+
const msg = entry.message as AssistantMessage;
|
|
177
|
+
if (msg.model && msg.provider) {
|
|
178
|
+
// Emit unconditionally. The aggregator's UPDATE is guarded by
|
|
179
|
+
// `model IS NULL` so this is idempotent: a no-op for already
|
|
180
|
+
// linked rows, a fix-up for fresh inserts (which start NULL
|
|
181
|
+
// because the user row is recorded before its reply lands) and
|
|
182
|
+
// for cross-pass orphans whose parent was committed by an
|
|
183
|
+
// earlier incremental sync.
|
|
184
|
+
userLinks.push({
|
|
185
|
+
sessionFile: sessionPath,
|
|
186
|
+
entryId: parentId,
|
|
187
|
+
model: msg.model,
|
|
188
|
+
provider: msg.provider,
|
|
189
|
+
});
|
|
173
190
|
}
|
|
174
191
|
}
|
|
175
192
|
}
|
|
176
193
|
}
|
|
177
194
|
|
|
178
|
-
return { stats, userStats, newOffset: start + read };
|
|
195
|
+
return { stats, userStats, userLinks, newOffset: start + read };
|
|
179
196
|
}
|
|
180
197
|
|
|
181
198
|
/**
|
|
@@ -0,0 +1,31 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Stateless parse worker for `syncAllSessions`. The main thread owns the
|
|
3
|
+
* SQLite handle; workers receive `{ sessionFile, fromOffset }`, run
|
|
4
|
+
* `parseSessionFile` (which is pure I/O + CPU, no DB), and post the
|
|
5
|
+
* structured-clone-safe result back. One in-flight request per worker so
|
|
6
|
+
* the main thread can fan jobs out 1:1 with the pool size.
|
|
7
|
+
*/
|
|
8
|
+
|
|
9
|
+
import { type ParseSessionResult, parseSessionFile } from "./parser";
|
|
10
|
+
|
|
11
|
+
export interface SyncWorkerRequest {
|
|
12
|
+
sessionFile: string;
|
|
13
|
+
fromOffset: number;
|
|
14
|
+
}
|
|
15
|
+
|
|
16
|
+
export type SyncWorkerResponse = { ok: true; result: ParseSessionResult } | { ok: false; error: string };
|
|
17
|
+
|
|
18
|
+
declare const self: Worker & {
|
|
19
|
+
onmessage: ((event: MessageEvent<SyncWorkerRequest>) => void) | null;
|
|
20
|
+
};
|
|
21
|
+
|
|
22
|
+
self.onmessage = async event => {
|
|
23
|
+
const { sessionFile, fromOffset } = event.data;
|
|
24
|
+
try {
|
|
25
|
+
const result = await parseSessionFile(sessionFile, fromOffset);
|
|
26
|
+
self.postMessage({ ok: true, result } satisfies SyncWorkerResponse);
|
|
27
|
+
} catch (err) {
|
|
28
|
+
const error = err instanceof Error ? (err.stack ?? err.message) : String(err);
|
|
29
|
+
self.postMessage({ ok: false, error } satisfies SyncWorkerResponse);
|
|
30
|
+
}
|
|
31
|
+
};
|
package/src/types.ts
CHANGED
|
@@ -219,11 +219,31 @@ export interface UserMessageStats {
|
|
|
219
219
|
/** Whitespace-delimited word count */
|
|
220
220
|
words: number;
|
|
221
221
|
/** Yelling sentences (> 50% uppercase letters) */
|
|
222
|
-
|
|
222
|
+
yelling: number;
|
|
223
223
|
/** Profanity hits */
|
|
224
224
|
profanity: number;
|
|
225
|
-
/**
|
|
226
|
-
|
|
225
|
+
/** Catch-all upset signal: drama runs + `noooo`/`ughh`/... + `dude` + `..` */
|
|
226
|
+
anguish: number;
|
|
227
|
+
/** Corrective negation ("no", "nope", "thats not what i meant") */
|
|
228
|
+
negation: number;
|
|
229
|
+
/** User repeating themselves ("i meant", "still doesnt work", "like i said") */
|
|
230
|
+
repetition: number;
|
|
231
|
+
/** Second-person reproach ("you didnt", "you broke", "stop X-ing") */
|
|
232
|
+
blame: number;
|
|
233
|
+
}
|
|
234
|
+
|
|
235
|
+
/**
|
|
236
|
+
* Pair emitted by the parser when it sees an assistant message whose
|
|
237
|
+
* `parentId` points to a user message that wasn't parsed in the same pass
|
|
238
|
+
* (e.g. user prompt landed in an earlier incremental sync). The aggregator
|
|
239
|
+
* applies the link to the persisted `user_messages` row so it stops showing
|
|
240
|
+
* up in the "unknown" model bucket.
|
|
241
|
+
*/
|
|
242
|
+
export interface UserMessageLink {
|
|
243
|
+
sessionFile: string;
|
|
244
|
+
entryId: string;
|
|
245
|
+
model: string;
|
|
246
|
+
provider: string;
|
|
227
247
|
}
|
|
228
248
|
|
|
229
249
|
/**
|
|
@@ -239,20 +259,29 @@ export interface BehaviorTimeSeriesPoint {
|
|
|
239
259
|
/** Number of user messages in bucket */
|
|
240
260
|
messages: number;
|
|
241
261
|
/** Total yelling sentences in bucket */
|
|
242
|
-
|
|
262
|
+
yelling: number;
|
|
243
263
|
/** Total profanity hits in bucket */
|
|
244
264
|
profanity: number;
|
|
245
|
-
/** Total
|
|
246
|
-
|
|
265
|
+
/** Total anguish signal in bucket */
|
|
266
|
+
anguish: number;
|
|
267
|
+
/** Total corrective-negation hits in bucket */
|
|
268
|
+
negation: number;
|
|
269
|
+
/** Total user-repeating-themselves hits in bucket */
|
|
270
|
+
repetition: number;
|
|
271
|
+
/** Total second-person blame hits in bucket */
|
|
272
|
+
blame: number;
|
|
247
273
|
/** Total characters in bucket */
|
|
248
274
|
chars: number;
|
|
249
275
|
}
|
|
250
276
|
|
|
251
277
|
export interface BehaviorOverallStats {
|
|
252
278
|
totalMessages: number;
|
|
253
|
-
|
|
279
|
+
totalYelling: number;
|
|
254
280
|
totalProfanity: number;
|
|
255
|
-
|
|
281
|
+
totalAnguish: number;
|
|
282
|
+
totalNegation: number;
|
|
283
|
+
totalRepetition: number;
|
|
284
|
+
totalBlame: number;
|
|
256
285
|
totalChars: number;
|
|
257
286
|
firstTimestamp: number;
|
|
258
287
|
lastTimestamp: number;
|
|
@@ -265,9 +294,12 @@ export interface BehaviorModelStats {
|
|
|
265
294
|
model: string;
|
|
266
295
|
provider: string;
|
|
267
296
|
totalMessages: number;
|
|
268
|
-
|
|
297
|
+
totalYelling: number;
|
|
269
298
|
totalProfanity: number;
|
|
270
|
-
|
|
299
|
+
totalAnguish: number;
|
|
300
|
+
totalNegation: number;
|
|
301
|
+
totalRepetition: number;
|
|
302
|
+
totalBlame: number;
|
|
271
303
|
totalChars: number;
|
|
272
304
|
lastTimestamp: number;
|
|
273
305
|
}
|