@unblocklabs/unblock-memory 0.9.0 → 0.10.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +2 -2
- package/dist/src/config.js +6 -3
- package/dist/src/diagnostics.d.ts +2 -2
- package/dist/src/emotion-baselines.d.ts +120 -0
- package/dist/src/emotion-baselines.js +387 -0
- package/dist/src/inside-out-runtime.js +58 -9
- package/dist/src/inside-out-store.d.ts +37 -0
- package/dist/src/inside-out-store.js +137 -0
- package/dist/src/inside-out.d.ts +142 -7
- package/dist/src/inside-out.js +40 -62
- package/dist/src/manager.d.ts +2 -2
- package/dist/src/retrieval-telemetry.d.ts +1 -1
- package/docs/configuration.md +116 -10
- package/docs/peoplesql.md +7 -0
- package/docs/retrieval.md +5 -3
- package/openclaw.plugin.json +5 -4
- package/package.json +1 -1
package/README.md
CHANGED
|
@@ -110,7 +110,7 @@ Everything below is off until you configure it.
|
|
|
110
110
|
| Feature | What it does |
|
|
111
111
|
| --- | --- |
|
|
112
112
|
| [Session indexing](docs/retrieval.md#sessions) | Makes past conversations searchable. Group and channel chats by default; DMs only if you add them. |
|
|
113
|
-
| [Skill Whisperer](docs/retrieval.md#skill-whisperer) | Suggests one relevant skill for a user message. Never runs it.
|
|
113
|
+
| [Skill Whisperer](docs/retrieval.md#skill-whisperer) | Suggests one relevant skill for a user message. Never runs it. Turns on when you add a `skills` corpus; works without TypeSafe. |
|
|
114
114
|
| [Memory Whisperer](docs/retrieval.md#memory-whisperer) | Adds up to two relevant past excerpts before the agent replies. Needs TypeSafe and a self-hosted query-generation API. |
|
|
115
115
|
| [Safety Whisperer](docs/configuration.md#safety-whisperer) | Reminds the agent not to share sensitive topics, such as one person's salary, with requestors you haven't approved. A prompt-level guardrail, not access control; fails open if TypeSafe is unavailable. |
|
|
116
116
|
| [People dossiers](docs/peoplesql.md) | Lets the agent keep a short, evidence-backed note about each person it works with. |
|
|
@@ -120,7 +120,7 @@ Everything below is off until you configure it.
|
|
|
120
120
|
| [Clustering and curation](docs/retrieval.md#memory-analysis) | Groups related memories so the agent can spot duplicates and maintain its knowledge. Needs a local analysis worker. |
|
|
121
121
|
| [Quality and claim review](docs/retrieval.md#review-and-diagnostics) | Flags noisy indexed content and checks whether a claim is supported by its sources. Advisory only. |
|
|
122
122
|
| [Response audit](docs/response-audit.md) | Operator-only grading of reply quality and sentiment in approved Slack conversations. |
|
|
123
|
-
| [Inside Out](docs/configuration.md#inside-out) | Operator-only scores for six emotions in human replies, with reports. Never changes memory or prompts. |
|
|
123
|
+
| [Inside Out](docs/configuration.md#inside-out) | Operator-only scores for six emotions in human replies, compared with each person's own weekly baseline, with reports. Never changes memory or prompts. |
|
|
124
124
|
|
|
125
125
|
Whisperers also need conversation-hook and prompt-injection permissions on the
|
|
126
126
|
plugin entry; see [host controls](docs/configuration.md#host-controls). Each
|
package/dist/src/config.js
CHANGED
|
@@ -78,7 +78,7 @@ function resolveTypeSafe(value) {
|
|
|
78
78
|
};
|
|
79
79
|
}
|
|
80
80
|
const DEFAULT_SKILL_WHISPERER = {
|
|
81
|
-
enabled:
|
|
81
|
+
enabled: true,
|
|
82
82
|
historyMessages: 5,
|
|
83
83
|
minScore: 0.5,
|
|
84
84
|
cooldownTurns: 10,
|
|
@@ -345,7 +345,7 @@ export function resolveConfig(value) {
|
|
|
345
345
|
}
|
|
346
346
|
const value = config.skillWhisperer;
|
|
347
347
|
assertOnlyKeys(value, ["enabled", "historyMessages", "minScore", "cooldownTurns"], "skillWhisperer");
|
|
348
|
-
const enabled = value.enabled ??
|
|
348
|
+
const enabled = value.enabled ?? DEFAULT_SKILL_WHISPERER.enabled;
|
|
349
349
|
const historyMessages = value.historyMessages ?? 5;
|
|
350
350
|
const minScore = value.minScore ?? 0.5;
|
|
351
351
|
const cooldownTurns = value.cooldownTurns ?? 10;
|
|
@@ -369,8 +369,11 @@ export function resolveConfig(value) {
|
|
|
369
369
|
}
|
|
370
370
|
skillWhisperer = { enabled, historyMessages, minScore, cooldownTurns };
|
|
371
371
|
}
|
|
372
|
+
// Default-on means "on once skills exist": without a skills corpus it could never
|
|
373
|
+
// hint, so it registers no prompt hook and reports inactive. Host-applied schema
|
|
374
|
+
// defaults are indistinguishable from an explicit true, so this never throws.
|
|
372
375
|
if (skillWhisperer.enabled && !corpora.some((corpus) => corpus.kind === "skills")) {
|
|
373
|
-
|
|
376
|
+
skillWhisperer = { ...skillWhisperer, enabled: false };
|
|
374
377
|
}
|
|
375
378
|
return { corpora, keepEmbeddingModelWarm, analysis: analysisConfig, people, peoplePrimer, skillWhisperer,
|
|
376
379
|
extraction: resolveExtraction(config.extraction, corpora),
|
|
@@ -95,8 +95,8 @@ export declare class WhispererDiagnostics {
|
|
|
95
95
|
[k: string]: {
|
|
96
96
|
calls: number;
|
|
97
97
|
outcomes: {
|
|
98
|
-
ok?: number | undefined;
|
|
99
98
|
cancelled?: number | undefined;
|
|
99
|
+
ok?: number | undefined;
|
|
100
100
|
partial?: number | undefined;
|
|
101
101
|
skipped?: number | undefined;
|
|
102
102
|
failed?: number | undefined;
|
|
@@ -104,7 +104,7 @@ export declare class WhispererDiagnostics {
|
|
|
104
104
|
timed_out?: number | undefined;
|
|
105
105
|
};
|
|
106
106
|
slowestRecent: {
|
|
107
|
-
outcome: "
|
|
107
|
+
outcome: "cancelled" | "ok" | "partial" | "skipped" | "failed" | "empty" | "timed_out";
|
|
108
108
|
setupMs?: number | undefined;
|
|
109
109
|
queueWaitMs?: number | undefined;
|
|
110
110
|
metadataMs?: number | undefined;
|
|
@@ -0,0 +1,120 @@
|
|
|
1
|
+
import type { DatabaseSync, SQLOutputValue } from "node:sqlite";
|
|
2
|
+
export declare const INSIDE_OUT_EMOTIONS: readonly ["joy", "sadness", "fear", "anger", "disgust", "surprise"];
|
|
3
|
+
/** Erased by an operator purge: never retried, linked, aggregated or compared. */
|
|
4
|
+
export declare const PURGED_GRADE = "Purged by operator";
|
|
5
|
+
type Emotion = typeof INSIDE_OUT_EMOTIONS[number];
|
|
6
|
+
type Row = Record<string, SQLOutputValue>;
|
|
7
|
+
type Statistics = {
|
|
8
|
+
count: number;
|
|
9
|
+
mean: number | null;
|
|
10
|
+
min: number | null;
|
|
11
|
+
p10: number | null;
|
|
12
|
+
median: number | null;
|
|
13
|
+
p90: number | null;
|
|
14
|
+
max: number | null;
|
|
15
|
+
distribution: [number, number][];
|
|
16
|
+
};
|
|
17
|
+
type Summary = Record<Emotion, Statistics>;
|
|
18
|
+
type Deviation = {
|
|
19
|
+
delta: number;
|
|
20
|
+
percentile: number;
|
|
21
|
+
unusualness: number;
|
|
22
|
+
};
|
|
23
|
+
type Comparison = {
|
|
24
|
+
status: "ok" | "insufficient_history";
|
|
25
|
+
count: number;
|
|
26
|
+
emotions: Record<Emotion, Deviation> | null;
|
|
27
|
+
};
|
|
28
|
+
type PersonalDeviation = {
|
|
29
|
+
version: string;
|
|
30
|
+
status: "ok" | "insufficient_history" | "unlinked_person" | "missing_time" | "future_time" | "grade_failed";
|
|
31
|
+
personId: string | null;
|
|
32
|
+
baselineId: string | null;
|
|
33
|
+
baselineEnd: number | null;
|
|
34
|
+
assessedAt: number;
|
|
35
|
+
minInteractions: number;
|
|
36
|
+
cumulative: Comparison | null;
|
|
37
|
+
weekly: Comparison | null;
|
|
38
|
+
};
|
|
39
|
+
/**
|
|
40
|
+
* One current, content-addressed snapshot per person, rubric, model and completed week.
|
|
41
|
+
* Reads finish before each batch commits, so the shared store is never held for a whole pass.
|
|
42
|
+
* No inference and no changes to raw grades.
|
|
43
|
+
*/
|
|
44
|
+
export declare function refreshEmotionBaselines(db: DatabaseSync, options: {
|
|
45
|
+
minInteractions: number;
|
|
46
|
+
now?: number;
|
|
47
|
+
recompare?: boolean;
|
|
48
|
+
}): {
|
|
49
|
+
snapshots: number;
|
|
50
|
+
comparisons: number;
|
|
51
|
+
};
|
|
52
|
+
/** Prepared once per export, not per row. */
|
|
53
|
+
export declare function deviationReader(db: DatabaseSync): (row: Row, history?: boolean) => PersonalDeviation | {
|
|
54
|
+
status: string;
|
|
55
|
+
} | {
|
|
56
|
+
history: {
|
|
57
|
+
currentGrade: boolean;
|
|
58
|
+
version: string;
|
|
59
|
+
status: "ok" | "insufficient_history" | "unlinked_person" | "missing_time" | "future_time" | "grade_failed";
|
|
60
|
+
personId: string | null;
|
|
61
|
+
baselineId: string | null;
|
|
62
|
+
baselineEnd: number | null;
|
|
63
|
+
assessedAt: number;
|
|
64
|
+
minInteractions: number;
|
|
65
|
+
cumulative: Comparison | null;
|
|
66
|
+
weekly: Comparison | null;
|
|
67
|
+
}[];
|
|
68
|
+
version: string;
|
|
69
|
+
status: "ok" | "insufficient_history" | "unlinked_person" | "missing_time" | "future_time" | "grade_failed";
|
|
70
|
+
personId: string | null;
|
|
71
|
+
baselineId: string | null;
|
|
72
|
+
baselineEnd: number | null;
|
|
73
|
+
assessedAt: number;
|
|
74
|
+
minInteractions: number;
|
|
75
|
+
cumulative: Comparison | null;
|
|
76
|
+
weekly: Comparison | null;
|
|
77
|
+
} | {
|
|
78
|
+
history: {
|
|
79
|
+
currentGrade: boolean;
|
|
80
|
+
version: string;
|
|
81
|
+
status: "ok" | "insufficient_history" | "unlinked_person" | "missing_time" | "future_time" | "grade_failed";
|
|
82
|
+
personId: string | null;
|
|
83
|
+
baselineId: string | null;
|
|
84
|
+
baselineEnd: number | null;
|
|
85
|
+
assessedAt: number;
|
|
86
|
+
minInteractions: number;
|
|
87
|
+
cumulative: Comparison | null;
|
|
88
|
+
weekly: Comparison | null;
|
|
89
|
+
}[];
|
|
90
|
+
status: "not_compared";
|
|
91
|
+
};
|
|
92
|
+
/** Weekly snapshots, newest first; cumulative statistics are prefix sums of each population's stored weeks. */
|
|
93
|
+
export declare function emotionBaselineHistory(db: DatabaseSync, options?: {
|
|
94
|
+
personId?: string;
|
|
95
|
+
since?: number;
|
|
96
|
+
limit?: number;
|
|
97
|
+
}): {
|
|
98
|
+
id: string;
|
|
99
|
+
person_id: string;
|
|
100
|
+
rubric: string;
|
|
101
|
+
model: string;
|
|
102
|
+
version: string;
|
|
103
|
+
week_start: number;
|
|
104
|
+
week_end: number;
|
|
105
|
+
computed_at: number;
|
|
106
|
+
weekly: Summary;
|
|
107
|
+
cumulative: Summary;
|
|
108
|
+
}[];
|
|
109
|
+
/**
|
|
110
|
+
* Delete one person's baselines and assessment history. With `grades`, also erase their six
|
|
111
|
+
* probabilities and person link, keeping a tombstone so the reply is never sent to Jev again.
|
|
112
|
+
*/
|
|
113
|
+
export declare function purgeEmotionProfile(db: DatabaseSync, personId: string, options?: {
|
|
114
|
+
grades?: boolean;
|
|
115
|
+
}): {
|
|
116
|
+
baselines: number;
|
|
117
|
+
assessments: number;
|
|
118
|
+
grades: number;
|
|
119
|
+
};
|
|
120
|
+
export {};
|
|
@@ -0,0 +1,387 @@
|
|
|
1
|
+
import { createHash } from "node:crypto";
|
|
2
|
+
export const INSIDE_OUT_EMOTIONS = ["joy", "sadness", "fear", "anger", "disgust", "surprise"];
|
|
3
|
+
/** Erased by an operator purge: never retried, linked, aggregated or compared. */
|
|
4
|
+
export const PURGED_GRADE = "Purged by operator";
|
|
5
|
+
const WEEK = 7 * 86400_000;
|
|
6
|
+
const MONDAY = Date.UTC(1970, 0, 5);
|
|
7
|
+
const VERSION = "personal-deviation-v2";
|
|
8
|
+
// 0.01-wide bins bound each stored distribution to 101 values per emotion, however long the history.
|
|
9
|
+
const STEPS = 100;
|
|
10
|
+
// Writers on the shared store wait for at most one short batch, never a whole refresh.
|
|
11
|
+
const BATCH = 500;
|
|
12
|
+
const PAGE = 2000;
|
|
13
|
+
const hash = (value) => createHash("sha256").update(JSON.stringify(value)).digest("hex");
|
|
14
|
+
const weekStart = (time) => MONDAY + Math.floor((time - MONDAY) / WEEK) * WEEK;
|
|
15
|
+
const bin = (value) => Math.round(value * STEPS);
|
|
16
|
+
const tableExists = (db, name) => Boolean(db.prepare("SELECT 1 FROM sqlite_schema WHERE type='table' AND name=?").get(name));
|
|
17
|
+
const VALID = INSIDE_OUT_EMOTIONS.map(e => `e.${e} BETWEEN 0 AND 1`).join(" AND ");
|
|
18
|
+
// Comparable grades: successful, timed, not after `now`. Callers add the People join.
|
|
19
|
+
const COMPARABLE = `e.reviewed_at<=? AND e.error IS NULL AND e.target_at IS NOT NULL AND e.target_at<=? AND ${VALID}`;
|
|
20
|
+
function schemaVersion(db) {
|
|
21
|
+
if (!tableExists(db, "memory_schema"))
|
|
22
|
+
return undefined;
|
|
23
|
+
const version = db.prepare("SELECT version FROM memory_schema WHERE component='emotion-baselines'").get()?.version;
|
|
24
|
+
return version === undefined ? undefined : Number(version);
|
|
25
|
+
}
|
|
26
|
+
/** Separate analytics tables: never dossier fields or prompt-injection state. */
|
|
27
|
+
function initializeEmotionBaselines(db) {
|
|
28
|
+
if (schemaVersion(db) === 2)
|
|
29
|
+
return;
|
|
30
|
+
db.exec("BEGIN IMMEDIATE");
|
|
31
|
+
try {
|
|
32
|
+
const version = schemaVersion(db);
|
|
33
|
+
if (version !== undefined && version !== 1 && version !== 2)
|
|
34
|
+
throw new Error("Unsupported Inside Out baseline storage version");
|
|
35
|
+
// Unreleased v1 kept every raw-value revision. Both tables are derived and rebuild from grades.
|
|
36
|
+
if (version === 1)
|
|
37
|
+
db.exec("DROP TABLE IF EXISTS inside_out_deviations; DROP TABLE IF EXISTS person_emotion_baselines;");
|
|
38
|
+
db.exec(`CREATE TABLE IF NOT EXISTS person_emotion_baselines (
|
|
39
|
+
person_id TEXT NOT NULL, rubric TEXT NOT NULL, model TEXT NOT NULL, week_start INTEGER NOT NULL,
|
|
40
|
+
id TEXT NOT NULL, version TEXT NOT NULL, week_end INTEGER NOT NULL, computed_at INTEGER NOT NULL, weekly_json TEXT NOT NULL,
|
|
41
|
+
PRIMARY KEY(person_id,rubric,model,week_start)
|
|
42
|
+
) STRICT;
|
|
43
|
+
CREATE TABLE IF NOT EXISTS inside_out_deviations (
|
|
44
|
+
id TEXT PRIMARY KEY, interaction_id TEXT NOT NULL, rubric TEXT NOT NULL, grade_signature TEXT NOT NULL,
|
|
45
|
+
assessed_at INTEGER NOT NULL, baseline_id TEXT, person_id TEXT, result_json TEXT NOT NULL
|
|
46
|
+
) STRICT;
|
|
47
|
+
CREATE INDEX IF NOT EXISTS inside_out_deviations_interaction ON inside_out_deviations(interaction_id,rubric);
|
|
48
|
+
CREATE INDEX IF NOT EXISTS inside_out_deviations_person ON inside_out_deviations(person_id);
|
|
49
|
+
INSERT OR REPLACE INTO memory_schema VALUES ('emotion-baselines',2);`);
|
|
50
|
+
db.exec("COMMIT");
|
|
51
|
+
}
|
|
52
|
+
catch (error) {
|
|
53
|
+
db.exec("ROLLBACK");
|
|
54
|
+
throw error;
|
|
55
|
+
}
|
|
56
|
+
}
|
|
57
|
+
function gradeSignature(row) {
|
|
58
|
+
return hash([row.interaction_id, row.rubric, row.model, row.person_id ?? null, row.target_at,
|
|
59
|
+
row.reviewed_at, row.error, ...INSIDE_OUT_EMOTIONS.map(e => row[e])]);
|
|
60
|
+
}
|
|
61
|
+
function histograms() {
|
|
62
|
+
const empty = () => ({ count: 0, sum: 0, carry: 0, min: Infinity, max: -Infinity, bins: new Array(STEPS + 1).fill(0) });
|
|
63
|
+
return { joy: empty(), sadness: empty(), fear: empty(), anger: empty(), disgust: empty(), surprise: empty() };
|
|
64
|
+
}
|
|
65
|
+
// Neumaier summation keeps means exact for repeated values: twenty 0.8s average to 0.8.
|
|
66
|
+
function accumulate(target, value, count, min, max) {
|
|
67
|
+
const total = target.sum + value;
|
|
68
|
+
target.carry += Math.abs(target.sum) >= Math.abs(value) ? target.sum - total + value : value - total + target.sum;
|
|
69
|
+
target.sum = total;
|
|
70
|
+
target.count += count;
|
|
71
|
+
target.min = Math.min(target.min, min);
|
|
72
|
+
target.max = Math.max(target.max, max);
|
|
73
|
+
}
|
|
74
|
+
function add(target, row) {
|
|
75
|
+
for (const emotion of INSIDE_OUT_EMOTIONS) {
|
|
76
|
+
const value = Number(row[emotion]);
|
|
77
|
+
accumulate(target[emotion], value, 1, value, value);
|
|
78
|
+
target[emotion].bins[bin(value)]++;
|
|
79
|
+
}
|
|
80
|
+
}
|
|
81
|
+
function merge(target, source) {
|
|
82
|
+
for (const emotion of INSIDE_OUT_EMOTIONS) {
|
|
83
|
+
const from = source[emotion], to = target[emotion];
|
|
84
|
+
accumulate(to, from.sum + from.carry, from.count, from.min, from.max);
|
|
85
|
+
from.bins.forEach((n, i) => { to.bins[i] += n; });
|
|
86
|
+
}
|
|
87
|
+
}
|
|
88
|
+
const distribution = (histogram) => histogram.bins.flatMap((n, i) => n ? [[i / STEPS, n]] : []);
|
|
89
|
+
/** Stored form: exact count, sum, min and max with sparse bins, so quiet weeks stay tiny. */
|
|
90
|
+
function encode(week) {
|
|
91
|
+
return JSON.stringify(Object.fromEntries(INSIDE_OUT_EMOTIONS.map(emotion => {
|
|
92
|
+
const h = week[emotion];
|
|
93
|
+
return [emotion, { count: h.count, sum: h.sum + h.carry, min: h.count ? h.min : null, max: h.count ? h.max : null, bins: distribution(h) }];
|
|
94
|
+
})));
|
|
95
|
+
}
|
|
96
|
+
function decode(json) {
|
|
97
|
+
const stored = JSON.parse(json);
|
|
98
|
+
const week = histograms();
|
|
99
|
+
for (const emotion of INSIDE_OUT_EMOTIONS) {
|
|
100
|
+
const s = stored[emotion], h = week[emotion];
|
|
101
|
+
Object.assign(h, { count: s.count, sum: s.sum, min: s.min ?? Infinity, max: s.max ?? -Infinity });
|
|
102
|
+
for (const [value, n] of s.bins)
|
|
103
|
+
h.bins[bin(value)] = n;
|
|
104
|
+
}
|
|
105
|
+
return week;
|
|
106
|
+
}
|
|
107
|
+
function summarize(week) {
|
|
108
|
+
const statistics = (h) => {
|
|
109
|
+
const values = distribution(h);
|
|
110
|
+
const at = (rank) => {
|
|
111
|
+
let seen = 0;
|
|
112
|
+
for (const [value, frequency] of values) {
|
|
113
|
+
seen += frequency;
|
|
114
|
+
if (rank < seen)
|
|
115
|
+
return value;
|
|
116
|
+
}
|
|
117
|
+
return values.at(-1)[0];
|
|
118
|
+
};
|
|
119
|
+
// Bin centres can sit just outside the exact extremes; quantiles never do.
|
|
120
|
+
const quantile = (p) => {
|
|
121
|
+
if (!h.count)
|
|
122
|
+
return null;
|
|
123
|
+
const position = (h.count - 1) * p, lower = Math.floor(position);
|
|
124
|
+
return Math.min(h.max, Math.max(h.min, at(lower) + (at(Math.ceil(position)) - at(lower)) * (position - lower)));
|
|
125
|
+
};
|
|
126
|
+
return { count: h.count, mean: h.count ? (h.sum + h.carry) / h.count : null, min: h.count ? h.min : null,
|
|
127
|
+
p10: quantile(0.1), median: quantile(0.5), p90: quantile(0.9), max: h.count ? h.max : null, distribution: values };
|
|
128
|
+
};
|
|
129
|
+
return { joy: statistics(week.joy), sadness: statistics(week.sadness), fear: statistics(week.fear),
|
|
130
|
+
anger: statistics(week.anger), disgust: statistics(week.disgust), surprise: statistics(week.surprise) };
|
|
131
|
+
}
|
|
132
|
+
/** Freeze a population's rank prefixes before later grades join it. */
|
|
133
|
+
function ranks(population) {
|
|
134
|
+
const freeze = (h) => {
|
|
135
|
+
const prefix = [0];
|
|
136
|
+
for (const n of h.bins)
|
|
137
|
+
prefix.push(prefix.at(-1) + n);
|
|
138
|
+
return { count: h.count, mean: (h.sum + h.carry) / h.count, prefix };
|
|
139
|
+
};
|
|
140
|
+
return { joy: freeze(population.joy), sadness: freeze(population.sadness), fear: freeze(population.fear),
|
|
141
|
+
anger: freeze(population.anger), disgust: freeze(population.disgust), surprise: freeze(population.surprise) };
|
|
142
|
+
}
|
|
143
|
+
function compare(row, population, minimum) {
|
|
144
|
+
const count = population.joy.count;
|
|
145
|
+
if (count < minimum)
|
|
146
|
+
return { status: "insufficient_history", count, emotions: null };
|
|
147
|
+
const deviation = (emotion) => {
|
|
148
|
+
const score = Number(row[emotion]), { mean, prefix } = population[emotion], b = bin(score);
|
|
149
|
+
// Midrank keeps a tied, constant baseline at 0.5 rather than treating every tie as extreme.
|
|
150
|
+
const percentile = (prefix[b] + (prefix[b + 1] - prefix[b]) / 2) / count;
|
|
151
|
+
return { delta: score - mean, percentile, unusualness: Math.abs(2 * percentile - 1) };
|
|
152
|
+
};
|
|
153
|
+
return { status: "ok", count, emotions: { joy: deviation("joy"), sadness: deviation("sadness"), fear: deviation("fear"),
|
|
154
|
+
anger: deviation("anger"), disgust: deviation("disgust"), surprise: deviation("surprise") } };
|
|
155
|
+
}
|
|
156
|
+
/** Queue writes and commit them in short IMMEDIATE transactions. Only call between completed reads. */
|
|
157
|
+
function batches(db) {
|
|
158
|
+
const pending = [];
|
|
159
|
+
const flush = () => {
|
|
160
|
+
if (!pending.length)
|
|
161
|
+
return;
|
|
162
|
+
db.exec("BEGIN IMMEDIATE");
|
|
163
|
+
try {
|
|
164
|
+
for (const write of pending)
|
|
165
|
+
write();
|
|
166
|
+
db.exec("COMMIT");
|
|
167
|
+
}
|
|
168
|
+
catch (error) {
|
|
169
|
+
db.exec("ROLLBACK");
|
|
170
|
+
throw error;
|
|
171
|
+
}
|
|
172
|
+
pending.length = 0;
|
|
173
|
+
};
|
|
174
|
+
return { write(change) { pending.push(change); if (pending.length >= BATCH)
|
|
175
|
+
flush(); }, flush };
|
|
176
|
+
}
|
|
177
|
+
/**
|
|
178
|
+
* One current, content-addressed snapshot per person, rubric, model and completed week.
|
|
179
|
+
* Reads finish before each batch commits, so the shared store is never held for a whole pass.
|
|
180
|
+
* No inference and no changes to raw grades.
|
|
181
|
+
*/
|
|
182
|
+
export function refreshEmotionBaselines(db, options) {
|
|
183
|
+
const now = options.now ?? Date.now(), cutoff = weekStart(now), minimum = options.minInteractions;
|
|
184
|
+
if (!Number.isFinite(now) || !Number.isInteger(minimum) || minimum < 1)
|
|
185
|
+
throw new Error("Invalid baseline settings");
|
|
186
|
+
initializeEmotionBaselines(db);
|
|
187
|
+
let snapshots = 0, comparisons = 0;
|
|
188
|
+
const queue = batches(db), people = tableExists(db, "people");
|
|
189
|
+
const prior = db.prepare(`SELECT id,result_json FROM inside_out_deviations WHERE interaction_id=? AND rubric=? AND grade_signature=?
|
|
190
|
+
ORDER BY assessed_at DESC,rowid DESC LIMIT 1`);
|
|
191
|
+
const save = db.prepare("INSERT OR IGNORE INTO inside_out_deviations VALUES (?,?,?,?,?,?,?,?)");
|
|
192
|
+
const assess = (row, signature, decide) => {
|
|
193
|
+
const previous = prior.get(row.interaction_id, row.rubric, signature);
|
|
194
|
+
if (!options.recompare && previous)
|
|
195
|
+
return;
|
|
196
|
+
const result = { version: VERSION, status: "insufficient_history", personId: row.person_id === null ? null : String(row.person_id),
|
|
197
|
+
baselineId: null, baselineEnd: null, assessedAt: now, minInteractions: minimum, cumulative: null, weekly: null };
|
|
198
|
+
decide(result);
|
|
199
|
+
const key = (assessment) => hash([assessment.version, signature, assessment.baselineId, assessment.status, assessment.minInteractions]);
|
|
200
|
+
if (previous && key(JSON.parse(String(previous.result_json))) === key(result))
|
|
201
|
+
return;
|
|
202
|
+
// Chaining records permits an explicit return to an earlier baseline without
|
|
203
|
+
// selecting an obsolete intervening assessment or losing either observation.
|
|
204
|
+
const id = hash([key(result), previous?.id ?? null]);
|
|
205
|
+
queue.write(() => {
|
|
206
|
+
comparisons += Number(save.run(id, row.interaction_id, row.rubric, signature, now, result.baselineId, result.personId, JSON.stringify(result)).changes);
|
|
207
|
+
});
|
|
208
|
+
};
|
|
209
|
+
const populations = new Set();
|
|
210
|
+
if (people) {
|
|
211
|
+
const stored = db.prepare("SELECT week_start,id FROM person_emotion_baselines WHERE person_id=? AND rubric=? AND model=?");
|
|
212
|
+
const upsert = db.prepare(`INSERT INTO person_emotion_baselines VALUES (?,?,?,?,?,?,?,?,?)
|
|
213
|
+
ON CONFLICT(person_id,rubric,model,week_start) DO UPDATE SET id=excluded.id,version=excluded.version,
|
|
214
|
+
week_end=excluded.week_end,computed_at=excluded.computed_at,weekly_json=excluded.weekly_json`);
|
|
215
|
+
const remove = db.prepare("DELETE FROM person_emotion_baselines WHERE person_id=? AND rubric=? AND model=? AND week_start=? AND id=?");
|
|
216
|
+
const page = db.prepare(`SELECT interaction_id,rubric,model,person_id,target_at,reviewed_at,error,${INSIDE_OUT_EMOTIONS.join(",")}
|
|
217
|
+
FROM inside_out e WHERE e.person_id=? AND e.rubric=? AND e.model=? AND ${COMPARABLE}
|
|
218
|
+
AND (? IS NULL OR e.target_at>? OR (e.target_at=? AND e.interaction_id>?)) ORDER BY e.target_at,e.interaction_id LIMIT ?`);
|
|
219
|
+
const groups = db.prepare(`SELECT DISTINCT e.person_id,e.rubric,e.model FROM inside_out e JOIN people p ON p.id=e.person_id
|
|
220
|
+
WHERE ${COMPARABLE}`).all(now, now);
|
|
221
|
+
for (const group of groups) {
|
|
222
|
+
const { person_id: person, rubric, model } = group;
|
|
223
|
+
populations.add(JSON.stringify([person, rubric, model]));
|
|
224
|
+
const existing = new Map(stored.all(person, rubric, model).map(row => [Number(row.week_start), String(row.id)]));
|
|
225
|
+
function* rows() {
|
|
226
|
+
let last;
|
|
227
|
+
for (;;) {
|
|
228
|
+
const batch = page.all(person, rubric, model, now, now, last?.target_at ?? null, last?.target_at ?? null, last?.target_at ?? null, last?.interaction_id ?? null, PAGE);
|
|
229
|
+
yield* batch;
|
|
230
|
+
if (batch.length < PAGE)
|
|
231
|
+
return;
|
|
232
|
+
last = batch.at(-1);
|
|
233
|
+
}
|
|
234
|
+
}
|
|
235
|
+
const source = rows();
|
|
236
|
+
let next = source.next(), previous;
|
|
237
|
+
const cumulative = histograms(), current = new Set();
|
|
238
|
+
for (let start = next.done ? cutoff : weekStart(Number(next.value.target_at)); !next.done || start < cutoff; start += WEEK) {
|
|
239
|
+
const end = start + WEEK, weekly = histograms(), signatures = [];
|
|
240
|
+
// Grades compare only with completed weeks before their own; this week joins afterwards.
|
|
241
|
+
let frozen;
|
|
242
|
+
while (!next.done && Number(next.value.target_at) < end) {
|
|
243
|
+
const row = next.value, signature = gradeSignature(row);
|
|
244
|
+
assess(row, signature, result => {
|
|
245
|
+
if (!previous)
|
|
246
|
+
return;
|
|
247
|
+
frozen ??= { cumulative: ranks(cumulative), weekly: ranks(previous.weekly) };
|
|
248
|
+
result.baselineId = previous.id;
|
|
249
|
+
result.baselineEnd = previous.end;
|
|
250
|
+
result.cumulative = compare(row, frozen.cumulative, minimum);
|
|
251
|
+
result.weekly = compare(row, frozen.weekly, minimum);
|
|
252
|
+
result.status = result.cumulative.status;
|
|
253
|
+
});
|
|
254
|
+
if (start < cutoff) {
|
|
255
|
+
signatures.push(signature);
|
|
256
|
+
add(weekly, row);
|
|
257
|
+
}
|
|
258
|
+
next = source.next();
|
|
259
|
+
}
|
|
260
|
+
if (start >= cutoff)
|
|
261
|
+
continue;
|
|
262
|
+
merge(cumulative, weekly);
|
|
263
|
+
// Chained ids address the whole cumulative population without rehashing earlier weeks.
|
|
264
|
+
// Empty weeks are kept: a weekly comparison always means the preceding calendar week.
|
|
265
|
+
const id = hash([VERSION, person, rubric, model, start, previous?.id ?? null, signatures]), json = encode(weekly);
|
|
266
|
+
current.add(start);
|
|
267
|
+
if (existing.get(start) !== id)
|
|
268
|
+
queue.write(() => {
|
|
269
|
+
snapshots += Number(upsert.run(person, rubric, model, start, id, VERSION, end, now, json).changes);
|
|
270
|
+
});
|
|
271
|
+
previous = { id, end, weekly };
|
|
272
|
+
}
|
|
273
|
+
// Weeks no longer backed by linked grades are replaced, not retained as revisions.
|
|
274
|
+
for (const [start, id] of existing)
|
|
275
|
+
if (!current.has(start))
|
|
276
|
+
queue.write(() => { remove.run(person, rubric, model, start, id); });
|
|
277
|
+
}
|
|
278
|
+
}
|
|
279
|
+
// Populations without linked grades (removed identity or person) lose their snapshots.
|
|
280
|
+
const drop = db.prepare("DELETE FROM person_emotion_baselines WHERE person_id=? AND rubric=? AND model=?");
|
|
281
|
+
for (const row of db.prepare("SELECT DISTINCT person_id,rubric,model FROM person_emotion_baselines").all()) {
|
|
282
|
+
if (!populations.has(JSON.stringify([row.person_id, row.rubric, row.model])))
|
|
283
|
+
queue.write(() => { drop.run(row.person_id, row.rubric, row.model); });
|
|
284
|
+
}
|
|
285
|
+
// Everything not comparable records an explicit status, never a neutral score.
|
|
286
|
+
const comparable = people ? `coalesce(${COMPARABLE} AND EXISTS (SELECT 1 FROM people p WHERE p.id=e.person_id),0)` : "0";
|
|
287
|
+
const linked = people ? db.prepare("SELECT 1 FROM people WHERE id=?") : undefined;
|
|
288
|
+
const others = db.prepare(`SELECT rowid AS _rowid,* FROM inside_out e WHERE e.reviewed_at<=? AND e.rowid>?
|
|
289
|
+
AND (e.error IS NULL OR e.error<>?) AND NOT ${comparable} ORDER BY e.rowid LIMIT ?`);
|
|
290
|
+
for (let last = -Infinity;;) {
|
|
291
|
+
const batch = others.all(now, last, PURGED_GRADE, ...(people ? [now, now] : []), PAGE);
|
|
292
|
+
for (const row of batch) {
|
|
293
|
+
assess(row, gradeSignature(row), result => {
|
|
294
|
+
if (row.error !== null || !INSIDE_OUT_EMOTIONS.every(e => typeof row[e] === "number" && Number.isFinite(row[e]) && row[e] >= 0 && row[e] <= 1))
|
|
295
|
+
result.status = "grade_failed";
|
|
296
|
+
else if (!row.person_id || !linked?.get(row.person_id))
|
|
297
|
+
result.status = "unlinked_person";
|
|
298
|
+
else if (row.target_at === null)
|
|
299
|
+
result.status = "missing_time";
|
|
300
|
+
else
|
|
301
|
+
result.status = "future_time";
|
|
302
|
+
});
|
|
303
|
+
}
|
|
304
|
+
if (batch.length < PAGE)
|
|
305
|
+
break;
|
|
306
|
+
last = Number(batch.at(-1)._rowid);
|
|
307
|
+
queue.flush();
|
|
308
|
+
}
|
|
309
|
+
queue.flush();
|
|
310
|
+
return { snapshots, comparisons };
|
|
311
|
+
}
|
|
312
|
+
/** Prepared once per export, not per row. */
|
|
313
|
+
export function deviationReader(db) {
|
|
314
|
+
const entries = schemaVersion(db) === 2 ? db.prepare(`SELECT grade_signature,result_json FROM inside_out_deviations
|
|
315
|
+
WHERE interaction_id=? AND rubric=? ORDER BY assessed_at DESC,rowid DESC`) : undefined;
|
|
316
|
+
return (row, history = false) => {
|
|
317
|
+
if (!entries)
|
|
318
|
+
return { status: "not_compared" };
|
|
319
|
+
const all = entries.all(row.interaction_id, row.rubric);
|
|
320
|
+
const signature = gradeSignature(row), latest = all.find(entry => entry.grade_signature === signature);
|
|
321
|
+
const current = latest ? JSON.parse(String(latest.result_json)) : { status: "not_compared" };
|
|
322
|
+
return history ? { ...current, history: all.map(entry => {
|
|
323
|
+
const result = JSON.parse(String(entry.result_json));
|
|
324
|
+
return { ...result, currentGrade: entry.grade_signature === signature };
|
|
325
|
+
}) } : current;
|
|
326
|
+
};
|
|
327
|
+
}
|
|
328
|
+
/** Weekly snapshots, newest first; cumulative statistics are prefix sums of each population's stored weeks. */
|
|
329
|
+
export function emotionBaselineHistory(db, options = {}) {
|
|
330
|
+
const limit = options.limit ?? 100;
|
|
331
|
+
if (!Number.isInteger(limit) || limit < 1 || limit > 1000)
|
|
332
|
+
throw new Error("Baseline limit must be between 1 and 1000");
|
|
333
|
+
if (schemaVersion(db) !== 2)
|
|
334
|
+
return [];
|
|
335
|
+
const selected = db.prepare(`SELECT person_id,rubric,model,week_start FROM person_emotion_baselines
|
|
336
|
+
WHERE (? IS NULL OR person_id=?) AND (? IS NULL OR week_end>=?) ORDER BY week_end DESC,person_id,rubric,model LIMIT ?`)
|
|
337
|
+
.all(options.personId ?? null, options.personId ?? null, options.since ?? null, options.since ?? null, limit);
|
|
338
|
+
const wanted = new Map();
|
|
339
|
+
for (const row of selected) {
|
|
340
|
+
const population = JSON.stringify([row.person_id, row.rubric, row.model]);
|
|
341
|
+
wanted.set(population, (wanted.get(population) ?? new Set()).add(Number(row.week_start)));
|
|
342
|
+
}
|
|
343
|
+
const weeks = db.prepare("SELECT * FROM person_emotion_baselines WHERE person_id=? AND rubric=? AND model=? ORDER BY week_start");
|
|
344
|
+
const snapshots = new Map();
|
|
345
|
+
const snapshot = (row, weekly, cumulative) => ({ id: String(row.id), person_id: String(row.person_id),
|
|
346
|
+
rubric: String(row.rubric), model: String(row.model), version: String(row.version), week_start: Number(row.week_start),
|
|
347
|
+
week_end: Number(row.week_end), computed_at: Number(row.computed_at), weekly: summarize(weekly), cumulative: summarize(cumulative) });
|
|
348
|
+
for (const [population, starts] of wanted) {
|
|
349
|
+
const cumulative = histograms(), latest = Math.max(...starts);
|
|
350
|
+
for (const row of weeks.all(...JSON.parse(population))) {
|
|
351
|
+
const weekly = decode(String(row.weekly_json));
|
|
352
|
+
merge(cumulative, weekly);
|
|
353
|
+
if (starts.has(Number(row.week_start)))
|
|
354
|
+
snapshots.set(JSON.stringify([population, row.week_start]), snapshot(row, weekly, cumulative));
|
|
355
|
+
if (Number(row.week_start) >= latest)
|
|
356
|
+
break;
|
|
357
|
+
}
|
|
358
|
+
}
|
|
359
|
+
return selected.map(row => snapshots.get(JSON.stringify([JSON.stringify([row.person_id, row.rubric, row.model]), row.week_start])));
|
|
360
|
+
}
|
|
361
|
+
/**
|
|
362
|
+
* Delete one person's baselines and assessment history. With `grades`, also erase their six
|
|
363
|
+
* probabilities and person link, keeping a tombstone so the reply is never sent to Jev again.
|
|
364
|
+
*/
|
|
365
|
+
export function purgeEmotionProfile(db, personId, options = {}) {
|
|
366
|
+
const counts = { baselines: 0, assessments: 0, grades: 0 };
|
|
367
|
+
if (!tableExists(db, "inside_out"))
|
|
368
|
+
return counts;
|
|
369
|
+
initializeEmotionBaselines(db);
|
|
370
|
+
db.exec("BEGIN IMMEDIATE");
|
|
371
|
+
try {
|
|
372
|
+
if (options.grades) {
|
|
373
|
+
counts.assessments += Number(db.prepare(`DELETE FROM inside_out_deviations WHERE (interaction_id,rubric) IN
|
|
374
|
+
(SELECT interaction_id,rubric FROM inside_out WHERE person_id=?)`).run(personId).changes);
|
|
375
|
+
counts.grades = Number(db.prepare(`UPDATE inside_out SET ${INSIDE_OUT_EMOTIONS.map(e => `${e}=NULL`).join(",")},
|
|
376
|
+
error=?,person_id=NULL WHERE person_id=?`).run(PURGED_GRADE, personId).changes);
|
|
377
|
+
}
|
|
378
|
+
counts.assessments += Number(db.prepare("DELETE FROM inside_out_deviations WHERE person_id=?").run(personId).changes);
|
|
379
|
+
counts.baselines = Number(db.prepare("DELETE FROM person_emotion_baselines WHERE person_id=?").run(personId).changes);
|
|
380
|
+
db.exec("COMMIT");
|
|
381
|
+
return counts;
|
|
382
|
+
}
|
|
383
|
+
catch (error) {
|
|
384
|
+
db.exec("ROLLBACK");
|
|
385
|
+
throw error;
|
|
386
|
+
}
|
|
387
|
+
}
|