@c4t4/heyamigo 0.9.8 → 0.9.10

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/dist/boot.js CHANGED
@@ -9,6 +9,7 @@ import { attachIncoming } from './gateway/incoming.js';
9
9
  import { logger } from './logger.js';
10
10
  import { startScheduler } from './memory/scheduler.js';
11
11
  import { startChatWorkers, stopChatWorkers } from './queue/chat-worker.js';
12
+ import { startMemoryWorker, stopMemoryWorker, } from './queue/memory-worker.js';
12
13
  import { requestShutdown, startOrchestrator, stopOrchestrator, } from './queue/orchestrator.js';
13
14
  import { startSenderWorker, stopSenderWorker } from './queue/sender-worker.js';
14
15
  import { startSocket } from './wa/socket.js';
@@ -31,14 +32,16 @@ export async function bootBot() {
31
32
  onShutdownDrained: () => {
32
33
  stopChatWorkers();
33
34
  stopSenderWorker();
35
+ stopMemoryWorker();
34
36
  stopOrchestrator();
35
37
  closeDb();
36
38
  },
37
39
  });
38
- // Workers next. Inbound table is the source of truth; anything left
40
+ // Workers next. Queue tables are the source of truth; anything left
39
41
  // from a previous crash gets claimed by the new pool automatically.
40
42
  // No separate replay step needed.
41
43
  startSenderWorker();
44
+ startMemoryWorker();
42
45
  startChatWorkers();
43
46
  startScheduler();
44
47
  await startSocket((sock) => {
package/dist/cli/setup.js CHANGED
@@ -463,7 +463,10 @@ export async function runSetup() {
463
463
  }
464
464
  }
465
465
  // ── Storage ──────────────────────────────────────────────────
466
- run('mkdir -p storage/auth storage/messages storage/queue storage/prompts storage/media storage/outbox');
466
+ // storage/queue (the old fastq persistence dir) removed in Phase 4
467
+ // — inbound queue lives in SQLite now. storage/backups holds
468
+ // pre-migration snapshots; created on demand by the migration runner.
469
+ run('mkdir -p storage/auth storage/messages storage/prompts storage/media storage/outbox');
467
470
  run('mkdir -p storage/memory/buckets storage/memory/persons storage/memory/chats');
468
471
  p.log.success('Storage directories ready');
469
472
  // ── Import existing knowledge ────────────────────────────────
package/dist/db/schema.js CHANGED
@@ -185,3 +185,34 @@ export const inbound = sqliteTable('inbound', {
185
185
  .on(t.externalMsgId)
186
186
  .where(sql `${t.externalMsgId} IS NOT NULL`),
187
187
  }));
188
+ // ──────────────────────────────────────────────────────────────────
189
+ // memory_writes queue (Phase 5a)
190
+ // ──────────────────────────────────────────────────────────────────
191
+ // All memory mutations (journal append, journal create, digest
192
+ // trigger, compressed-view invalidation, future observe op) flow
193
+ // through here. One memory worker drains, serializing writes so
194
+ // parallel chat / async workers can't race on file edits.
195
+ //
196
+ // op + payload (JSON): operation-specific. The worker switches on op
197
+ // and calls the matching handler. Adding a new op = adding a switch
198
+ // arm + ensuring the handler is idempotent on re-delivery (since the
199
+ // queue retries on transient failure).
200
+ export const memoryWrites = sqliteTable('memory_writes', {
201
+ id: integer('id').primaryKey({ autoIncrement: true }),
202
+ op: text('op').notNull(), // 'append_journal'|'create_journal'|'trigger_digest'|'mark_compressed_dirty'|...
203
+ payload: text('payload').notNull(), // JSON
204
+ idempotencyKey: text('idempotency_key'),
205
+ status: text('status').notNull(), // 'pending'|'claimed'|'done'|'failed'|'dlq'
206
+ attempts: integer('attempts').notNull().default(0),
207
+ nextAttemptAt: integer('next_attempt_at'),
208
+ lastError: text('last_error'),
209
+ claimedBy: text('claimed_by'),
210
+ claimedAt: integer('claimed_at'),
211
+ createdAt: integer('created_at').notNull(),
212
+ updatedAt: integer('updated_at').notNull(),
213
+ }, t => ({
214
+ byStatusNext: index('memwr_by_status_next').on(t.status, t.nextAttemptAt),
215
+ uniqIdemp: uniqueIndex('memwr_idemp_uq')
216
+ .on(t.idempotencyKey)
217
+ .where(sql `${t.idempotencyKey} IS NOT NULL`),
218
+ }));
@@ -138,48 +138,47 @@ async function executeAsyncTask(task) {
138
138
  });
139
139
  }
140
140
  }
141
- // Journal creates run first so an entry flagged in the same output against
142
- // a new slug lands correctly.
143
- const { appendEntry, createJournal, getJournal, isValidSlug } = await import('../memory/journals.js');
144
- for (const op of journalCreates) {
141
+ // All memory mutations through memory_writes queue (Phase 5a).
142
+ const { enqueueMemoryWrite } = await import('./memory-writes.js');
143
+ const { isValidSlug } = await import('../memory/journals.js');
144
+ const memBase = `async-${task.id}`;
145
+ for (let i = 0; i < journalCreates.length; i++) {
146
+ const op = journalCreates[i];
145
147
  if (!isValidSlug(op.slug)) {
146
148
  logger.warn({ op, id: task.id }, 'async JOURNAL-NEW: invalid slug, dropped');
147
149
  continue;
148
150
  }
149
- if (getJournal(op.slug))
150
- continue;
151
- try {
152
- createJournal({
153
- slug: op.slug,
154
- name: titleCaseSlug(op.slug),
155
- purpose: op.purpose,
156
- });
157
- logger.info({ slug: op.slug, id: task.id }, 'journal created via async marker');
158
- }
159
- catch (err) {
160
- logger.error({ err, op, id: task.id }, 'async JOURNAL-NEW failed');
161
- }
151
+ enqueueMemoryWrite({
152
+ op: 'create_journal',
153
+ payload: { slug: op.slug, name: titleCaseSlug(op.slug), purpose: op.purpose },
154
+ idempotencyKey: `${memBase}-create-${i}`,
155
+ });
162
156
  }
157
+ // Treat enqueued appends as "appended" for the reporting line below;
158
+ // the memory worker logs the actual append+slug-validity outcomes.
163
159
  let appendedCount = 0;
164
- for (const j of journals) {
165
- const ok = appendEntry(j.slug, {
166
- source: 'async',
167
- jid: task.jid,
168
- senderNumber: task.senderNumber,
169
- note: j.note,
160
+ for (let i = 0; i < journals.length; i++) {
161
+ const j = journals[i];
162
+ enqueueMemoryWrite({
163
+ op: 'append_journal',
164
+ payload: {
165
+ slug: j.slug,
166
+ entry: {
167
+ source: 'async',
168
+ jid: task.jid,
169
+ senderNumber: task.senderNumber,
170
+ note: j.note,
171
+ },
172
+ },
173
+ idempotencyKey: `${memBase}-append-${i}`,
170
174
  });
171
- if (ok)
172
- appendedCount++;
173
- else {
174
- logger.warn({ slug: j.slug, id: task.id }, 'async JOURNAL marker pointed at unknown slug, dropped');
175
- }
175
+ appendedCount++;
176
176
  }
177
177
  if (digest) {
178
- const { scheduleDigest } = await import('../memory/scheduler.js');
179
- scheduleDigest({
180
- jid: task.jid,
181
- number: task.senderNumber,
182
- reason: digest,
178
+ enqueueMemoryWrite({
179
+ op: 'trigger_digest',
180
+ payload: { jid: task.jid, number: task.senderNumber, reason: digest },
181
+ idempotencyKey: `${memBase}-digest`,
183
182
  });
184
183
  }
185
184
  // The clean (marker-stripped) text IS the chat reply. Always send it when
@@ -428,41 +427,42 @@ async function runBrowserTask(task) {
428
427
  });
429
428
  }
430
429
  }
431
- const { appendEntry, createJournal, getJournal, isValidSlug } = await import('../memory/journals.js');
432
- for (const op of journalCreates) {
430
+ const { enqueueMemoryWrite } = await import('./memory-writes.js');
431
+ const { isValidSlug } = await import('../memory/journals.js');
432
+ const memBase = `browser-${task.id}`;
433
+ for (let i = 0; i < journalCreates.length; i++) {
434
+ const op = journalCreates[i];
433
435
  if (!isValidSlug(op.slug))
434
436
  continue;
435
- if (getJournal(op.slug))
436
- continue;
437
- try {
438
- createJournal({
439
- slug: op.slug,
440
- name: titleCaseSlug(op.slug),
441
- purpose: op.purpose,
442
- });
443
- logger.info({ slug: op.slug, id: task.id }, 'journal created via browser task marker');
444
- }
445
- catch (err) {
446
- logger.error({ err, op, id: task.id }, 'browser JOURNAL-NEW failed');
447
- }
437
+ enqueueMemoryWrite({
438
+ op: 'create_journal',
439
+ payload: { slug: op.slug, name: titleCaseSlug(op.slug), purpose: op.purpose },
440
+ idempotencyKey: `${memBase}-create-${i}`,
441
+ });
448
442
  }
449
443
  let appendedCount = 0;
450
- for (const j of journals) {
451
- const ok = appendEntry(j.slug, {
452
- source: 'async',
453
- jid: task.jid,
454
- senderNumber: task.senderNumber,
455
- note: j.note,
444
+ for (let i = 0; i < journals.length; i++) {
445
+ const j = journals[i];
446
+ enqueueMemoryWrite({
447
+ op: 'append_journal',
448
+ payload: {
449
+ slug: j.slug,
450
+ entry: {
451
+ source: 'async',
452
+ jid: task.jid,
453
+ senderNumber: task.senderNumber,
454
+ note: j.note,
455
+ },
456
+ },
457
+ idempotencyKey: `${memBase}-append-${i}`,
456
458
  });
457
- if (ok)
458
- appendedCount++;
459
+ appendedCount++;
459
460
  }
460
461
  if (digest) {
461
- const { scheduleDigest } = await import('../memory/scheduler.js');
462
- scheduleDigest({
463
- jid: task.jid,
464
- number: task.senderNumber,
465
- reason: digest,
462
+ enqueueMemoryWrite({
463
+ op: 'trigger_digest',
464
+ payload: { jid: task.jid, number: task.senderNumber, reason: digest },
465
+ idempotencyKey: `${memBase}-digest`,
466
466
  });
467
467
  }
468
468
  const chatText = clean.trim();
@@ -0,0 +1,169 @@
1
+ // Memory worker. Single concurrency by design: serializes ALL memory
2
+ // mutations into one writer thread so parallel chat / async workers
3
+ // can't race on file edits (full-file rewrites in particular).
4
+ //
5
+ // Dispatches by `op` to the existing handlers in src/memory/. The
6
+ // handlers themselves stay synchronous file writes — only the
7
+ // orchestration moved.
8
+ import { hostname } from 'os';
9
+ import { eq } from 'drizzle-orm';
10
+ import { getDb } from '../db/index.js';
11
+ import { workers } from '../db/schema.js';
12
+ import { logger } from '../logger.js';
13
+ import { claimNextMemoryWrite, markMemoryWriteDone, markMemoryWriteRetryOrDlq, } from './memory-writes.js';
14
+ const HEARTBEAT_INTERVAL_MS = 5_000;
15
+ const IDLE_POLL_INTERVAL_MS = 250;
16
+ let workerId = null;
17
+ let stopping = false;
18
+ let heartbeatTimer = null;
19
+ function newWorkerId() {
20
+ return `${hostname()}-${process.pid}-memory-0`;
21
+ }
22
+ function registerWorker(id) {
23
+ const db = getDb();
24
+ const now = Math.floor(Date.now() / 1000);
25
+ db.insert(workers)
26
+ .values({
27
+ id,
28
+ kind: 'memory',
29
+ status: 'idle',
30
+ currentJob: null,
31
+ lastSeen: now,
32
+ startedAt: now,
33
+ })
34
+ .onConflictDoUpdate({
35
+ target: workers.id,
36
+ set: { status: 'idle', currentJob: null, lastSeen: now, startedAt: now },
37
+ })
38
+ .run();
39
+ }
40
+ function setWorkerStatus(id, status, currentJob = null) {
41
+ const db = getDb();
42
+ db.update(workers)
43
+ .set({
44
+ status,
45
+ currentJob,
46
+ lastSeen: Math.floor(Date.now() / 1000),
47
+ })
48
+ .where(eq(workers.id, id))
49
+ .run();
50
+ }
51
+ function heartbeat(id) {
52
+ const db = getDb();
53
+ db.update(workers)
54
+ .set({ lastSeen: Math.floor(Date.now() / 1000) })
55
+ .where(eq(workers.id, id))
56
+ .run();
57
+ }
58
+ async function applyOp(row) {
59
+ const payload = JSON.parse(row.payload);
60
+ switch (row.op) {
61
+ case 'append_journal': {
62
+ const { appendEntry } = await import('../memory/journals.js');
63
+ const slug = payload.slug;
64
+ const entry = payload.entry;
65
+ const ok = appendEntry(slug, entry);
66
+ if (!ok) {
67
+ // Unknown slug — log + treat as done (no point retrying).
68
+ logger.warn({ slug }, 'memory_writes: append_journal slug not found, dropped');
69
+ }
70
+ return;
71
+ }
72
+ case 'create_journal': {
73
+ const { createJournal, getJournal, isValidSlug } = await import('../memory/journals.js');
74
+ const slug = payload.slug;
75
+ if (!isValidSlug(slug)) {
76
+ logger.warn({ slug }, 'memory_writes: create_journal invalid slug, dropped');
77
+ return;
78
+ }
79
+ if (getJournal(slug)) {
80
+ logger.info({ slug }, 'memory_writes: create_journal for existing slug, ignored');
81
+ return;
82
+ }
83
+ createJournal({
84
+ slug,
85
+ name: payload.name,
86
+ purpose: payload.purpose,
87
+ });
88
+ logger.info({ slug }, 'journal created via memory_writes');
89
+ return;
90
+ }
91
+ case 'trigger_digest': {
92
+ const { scheduleDigest } = await import('../memory/scheduler.js');
93
+ scheduleDigest({
94
+ jid: payload.jid,
95
+ number: payload.number,
96
+ reason: payload.reason,
97
+ });
98
+ return;
99
+ }
100
+ case 'mark_compressed_dirty': {
101
+ const { markCompressedDirty } = await import('../memory/compressed.js');
102
+ markCompressedDirty();
103
+ return;
104
+ }
105
+ default:
106
+ throw new Error(`unknown memory_writes op: ${row.op}`);
107
+ }
108
+ }
109
+ async function processOne(workerId, row) {
110
+ setWorkerStatus(workerId, 'busy', `memory_writes:${row.id}`);
111
+ try {
112
+ await applyOp(row);
113
+ const ok = markMemoryWriteDone(row.id, workerId);
114
+ if (!ok) {
115
+ logger.warn({ id: row.id, workerId }, 'memory_writes markDone failed (claim lost?). op already applied.');
116
+ }
117
+ }
118
+ catch (err) {
119
+ const msg = err instanceof Error ? err.message : String(err);
120
+ const result = markMemoryWriteRetryOrDlq(row.id, workerId, msg);
121
+ if (result.deadLettered) {
122
+ logger.error({ err, id: row.id, op: row.op }, 'memory_writes dead-lettered');
123
+ }
124
+ else if (result.retried) {
125
+ logger.warn({ err, id: row.id, op: row.op }, 'memory_writes transient fail, will retry');
126
+ }
127
+ }
128
+ finally {
129
+ setWorkerStatus(workerId, 'idle');
130
+ }
131
+ }
132
+ async function loop(workerId) {
133
+ while (!stopping) {
134
+ let processed = false;
135
+ try {
136
+ const row = claimNextMemoryWrite(workerId);
137
+ if (row) {
138
+ await processOne(workerId, row);
139
+ processed = true;
140
+ }
141
+ }
142
+ catch (err) {
143
+ logger.error({ err, workerId }, 'memory worker loop error');
144
+ }
145
+ if (!processed) {
146
+ await new Promise((res) => setTimeout(res, IDLE_POLL_INTERVAL_MS));
147
+ }
148
+ else {
149
+ await new Promise((res) => setImmediate(res));
150
+ }
151
+ }
152
+ setWorkerStatus(workerId, 'dead');
153
+ }
154
+ export function startMemoryWorker() {
155
+ if (workerId) {
156
+ logger.warn('memory worker already started; ignoring');
157
+ return;
158
+ }
159
+ workerId = newWorkerId();
160
+ registerWorker(workerId);
161
+ heartbeatTimer = setInterval(() => workerId && heartbeat(workerId), HEARTBEAT_INTERVAL_MS);
162
+ void loop(workerId).catch((err) => logger.fatal({ err }, 'memory worker loop crashed'));
163
+ logger.info({ workerId }, 'memory worker started');
164
+ }
165
+ export function stopMemoryWorker() {
166
+ stopping = true;
167
+ if (heartbeatTimer)
168
+ clearInterval(heartbeatTimer);
169
+ }
@@ -0,0 +1,136 @@
1
+ // memory_writes queue helpers. Producers (chat workers, async
2
+ // workers, journal observer, etc.) call enqueueMemoryWrite. The
3
+ // single memory worker drains. Op is a discriminator; payload is
4
+ // op-specific JSON.
5
+ import { and, asc, eq, isNull, lte, or, sql } from 'drizzle-orm';
6
+ import { getDb } from '../db/index.js';
7
+ import { memoryWrites } from '../db/schema.js';
8
+ export function enqueueMemoryWrite(input) {
9
+ const db = getDb();
10
+ const now = Math.floor(Date.now() / 1000);
11
+ if (input.idempotencyKey) {
12
+ const found = db
13
+ .select()
14
+ .from(memoryWrites)
15
+ .where(eq(memoryWrites.idempotencyKey, input.idempotencyKey))
16
+ .get();
17
+ if (found)
18
+ return { inserted: false, row: found };
19
+ }
20
+ const row = db
21
+ .insert(memoryWrites)
22
+ .values({
23
+ op: input.op,
24
+ payload: JSON.stringify(input.payload),
25
+ idempotencyKey: input.idempotencyKey ?? null,
26
+ status: 'pending',
27
+ attempts: 0,
28
+ nextAttemptAt: null,
29
+ lastError: null,
30
+ claimedBy: null,
31
+ claimedAt: null,
32
+ createdAt: now,
33
+ updatedAt: now,
34
+ })
35
+ .returning()
36
+ .get();
37
+ return { inserted: true, row };
38
+ }
39
+ export function claimNextMemoryWrite(workerId) {
40
+ const db = getDb();
41
+ const now = Math.floor(Date.now() / 1000);
42
+ return db.transaction((tx) => {
43
+ const target = tx
44
+ .select({ id: memoryWrites.id })
45
+ .from(memoryWrites)
46
+ .where(and(eq(memoryWrites.status, 'pending'), or(isNull(memoryWrites.nextAttemptAt), lte(memoryWrites.nextAttemptAt, now))))
47
+ .orderBy(asc(memoryWrites.id))
48
+ .limit(1)
49
+ .get();
50
+ if (!target)
51
+ return null;
52
+ const claimed = tx
53
+ .update(memoryWrites)
54
+ .set({
55
+ status: 'claimed',
56
+ claimedBy: workerId,
57
+ claimedAt: now,
58
+ updatedAt: now,
59
+ })
60
+ .where(and(eq(memoryWrites.id, target.id), eq(memoryWrites.status, 'pending')))
61
+ .returning()
62
+ .get();
63
+ return claimed ?? null;
64
+ });
65
+ }
66
+ export function markMemoryWriteDone(id, workerId) {
67
+ const db = getDb();
68
+ const now = Math.floor(Date.now() / 1000);
69
+ const result = db
70
+ .update(memoryWrites)
71
+ .set({ status: 'done', updatedAt: now })
72
+ .where(and(eq(memoryWrites.id, id), eq(memoryWrites.status, 'claimed'), eq(memoryWrites.claimedBy, workerId)))
73
+ .returning({ id: memoryWrites.id })
74
+ .all();
75
+ return result.length > 0;
76
+ }
77
+ // Memory mutations are cheap; if one fails we don't want to bury it
78
+ // in long backoffs. Quick retries, fast DLQ.
79
+ const BACKOFF_SECONDS = [1, 5, 30];
80
+ const MAX_ATTEMPTS = BACKOFF_SECONDS.length;
81
+ export function markMemoryWriteRetryOrDlq(id, workerId, errorMessage) {
82
+ const db = getDb();
83
+ return db.transaction((tx) => {
84
+ const row = tx.select().from(memoryWrites).where(eq(memoryWrites.id, id)).get();
85
+ if (!row || row.status !== 'claimed' || row.claimedBy !== workerId) {
86
+ return { retried: false, deadLettered: false };
87
+ }
88
+ const now = Math.floor(Date.now() / 1000);
89
+ const nextAttempts = row.attempts + 1;
90
+ if (nextAttempts > MAX_ATTEMPTS) {
91
+ tx.update(memoryWrites)
92
+ .set({
93
+ status: 'dlq',
94
+ attempts: nextAttempts,
95
+ lastError: errorMessage,
96
+ claimedBy: null,
97
+ claimedAt: null,
98
+ updatedAt: now,
99
+ })
100
+ .where(eq(memoryWrites.id, id))
101
+ .run();
102
+ return { retried: false, deadLettered: true };
103
+ }
104
+ const backoff = BACKOFF_SECONDS[Math.min(row.attempts, BACKOFF_SECONDS.length - 1)];
105
+ tx.update(memoryWrites)
106
+ .set({
107
+ status: 'pending',
108
+ attempts: nextAttempts,
109
+ nextAttemptAt: now + backoff,
110
+ lastError: errorMessage,
111
+ claimedBy: null,
112
+ claimedAt: null,
113
+ updatedAt: now,
114
+ })
115
+ .where(eq(memoryWrites.id, id))
116
+ .run();
117
+ return { retried: true, deadLettered: false };
118
+ });
119
+ }
120
+ const CLAIM_TTL_SECONDS = 60;
121
+ export function reclaimStuckMemoryWrites() {
122
+ const db = getDb();
123
+ const cutoff = Math.floor(Date.now() / 1000) - CLAIM_TTL_SECONDS;
124
+ const result = db
125
+ .update(memoryWrites)
126
+ .set({
127
+ status: 'pending',
128
+ claimedBy: null,
129
+ claimedAt: null,
130
+ updatedAt: sql `${memoryWrites.updatedAt}`,
131
+ })
132
+ .where(and(eq(memoryWrites.status, 'claimed'), lte(memoryWrites.claimedAt, cutoff)))
133
+ .returning({ id: memoryWrites.id })
134
+ .all();
135
+ return result.length;
136
+ }
@@ -17,6 +17,7 @@ import { getDb } from '../db/index.js';
17
17
  import { workers } from '../db/schema.js';
18
18
  import { logger } from '../logger.js';
19
19
  import { reclaimStuckInbound } from './inbound.js';
20
+ import { reclaimStuckMemoryWrites } from './memory-writes.js';
20
21
  import { reclaimStuckOutbound } from './outbound.js';
21
22
  import { clearControl, readControl, requestControl } from './control.js';
22
23
  import { listDueCrons, markCronFired } from './crons.js';
@@ -107,6 +108,10 @@ async function tick(id) {
107
108
  if (reclaimedInbound > 0) {
108
109
  logger.info({ reclaimed: reclaimedInbound }, 'reclaimed stuck inbound rows');
109
110
  }
111
+ const reclaimedMemWr = reclaimStuckMemoryWrites();
112
+ if (reclaimedMemWr > 0) {
113
+ logger.info({ reclaimed: reclaimedMemWr }, 'reclaimed stuck memory_writes rows');
114
+ }
110
115
  // Fire any due crons. Order: dispatch each in turn; if dispatch
111
116
  // throws (it shouldn't — dispatch swallows), the cron is NOT
112
117
  // marked fired and we'll retry on the next tick.
@@ -4,9 +4,9 @@ import { config } from '../config.js';
4
4
  import { logger } from '../logger.js';
5
5
  import { addDailyTokens } from '../store/usage.js';
6
6
  import { extractFlags } from '../memory/digest-flag.js';
7
- import { appendEntry, createJournal, getJournal, isValidSlug, } from '../memory/journals.js';
8
- import { scheduleDigest } from '../memory/scheduler.js';
7
+ import { isValidSlug } from '../memory/journals.js';
9
8
  import { enqueueAsyncTask, enqueueBrowserTask } from './async-tasks.js';
9
+ import { enqueueMemoryWrite } from './memory-writes.js';
10
10
  import { enqueueOutbound } from './outbound.js';
11
11
  function isStaleSessionError(err) {
12
12
  return (err instanceof Error &&
@@ -41,47 +41,50 @@ async function callClaude(job) {
41
41
  addDailyTokens(job.senderNumber, usage.inputTokens + usage.outputTokens);
42
42
  }
43
43
  const { clean, digest, journals, journalCreates, asyncTasks, asyncBrowserTasks, sendTexts, } = extractFlags(reply);
44
+ // All memory mutations go through the memory_writes queue so the
45
+ // single memory worker serializes file writes — safe under parallel
46
+ // chat workers. Idempotency keys derived from job + index so a
47
+ // retry doesn't duplicate.
48
+ const memBase = `chat-${job.jid}-${Date.now()}`;
44
49
  if (digest) {
45
50
  logger.info({ jid: job.jid, number: job.senderNumber, reason: digest }, 'DIGEST flag raised, scheduling');
46
- scheduleDigest({
47
- jid: job.jid,
48
- number: job.senderNumber,
49
- reason: digest,
51
+ enqueueMemoryWrite({
52
+ op: 'trigger_digest',
53
+ payload: { jid: job.jid, number: job.senderNumber, reason: digest },
54
+ idempotencyKey: `${memBase}-digest`,
50
55
  });
51
56
  }
52
57
  // Creates run BEFORE entry appends so that a reply creating a new journal
53
- // AND flagging its first entry in the same turn works correctly.
54
- for (const op of journalCreates) {
58
+ // AND flagging its first entry in the same turn works correctly. The
59
+ // memory worker enforces this ordering because it drains serially in
60
+ // insert order.
61
+ for (let i = 0; i < journalCreates.length; i++) {
62
+ const op = journalCreates[i];
55
63
  if (!isValidSlug(op.slug)) {
56
64
  logger.warn({ op, jid: job.jid }, 'JOURNAL-NEW: invalid slug, dropped');
57
65
  continue;
58
66
  }
59
- try {
60
- if (getJournal(op.slug)) {
61
- logger.info({ slug: op.slug }, 'JOURNAL-NEW for existing slug, ignored');
62
- continue;
63
- }
64
- createJournal({
65
- slug: op.slug,
66
- name: titleCase(op.slug),
67
- purpose: op.purpose,
68
- });
69
- logger.info({ slug: op.slug, jid: job.jid }, 'journal created via bot marker');
70
- }
71
- catch (err) {
72
- logger.error({ err, op, jid: job.jid }, 'JOURNAL-NEW failed');
73
- }
67
+ enqueueMemoryWrite({
68
+ op: 'create_journal',
69
+ payload: { slug: op.slug, name: titleCase(op.slug), purpose: op.purpose },
70
+ idempotencyKey: `${memBase}-create-${i}`,
71
+ });
74
72
  }
75
- for (const j of journals) {
76
- const ok = appendEntry(j.slug, {
77
- source: 'reactive',
78
- jid: job.jid,
79
- senderNumber: job.senderNumber,
80
- note: j.note,
73
+ for (let i = 0; i < journals.length; i++) {
74
+ const j = journals[i];
75
+ enqueueMemoryWrite({
76
+ op: 'append_journal',
77
+ payload: {
78
+ slug: j.slug,
79
+ entry: {
80
+ source: 'reactive',
81
+ jid: job.jid,
82
+ senderNumber: job.senderNumber,
83
+ note: j.note,
84
+ },
85
+ },
86
+ idempotencyKey: `${memBase}-append-${i}`,
81
87
  });
82
- if (!ok) {
83
- logger.warn({ slug: j.slug, jid: job.jid }, 'JOURNAL flag pointed at unknown slug, dropped');
84
- }
85
88
  }
86
89
  // Async tasks: Claude delegated to background workers. Chat reply above
87
90
  // is the user-facing ack. Two lanes:
@@ -0,0 +1,17 @@
1
+ CREATE TABLE `memory_writes` (
2
+ `id` integer PRIMARY KEY AUTOINCREMENT NOT NULL,
3
+ `op` text NOT NULL,
4
+ `payload` text NOT NULL,
5
+ `idempotency_key` text,
6
+ `status` text NOT NULL,
7
+ `attempts` integer DEFAULT 0 NOT NULL,
8
+ `next_attempt_at` integer,
9
+ `last_error` text,
10
+ `claimed_by` text,
11
+ `claimed_at` integer,
12
+ `created_at` integer NOT NULL,
13
+ `updated_at` integer NOT NULL
14
+ );
15
+ --> statement-breakpoint
16
+ CREATE INDEX `memwr_by_status_next` ON `memory_writes` (`status`,`next_attempt_at`);--> statement-breakpoint
17
+ CREATE UNIQUE INDEX `memwr_idemp_uq` ON `memory_writes` (`idempotency_key`) WHERE "memory_writes"."idempotency_key" IS NOT NULL;