@timqi/pier 0.1.0 → 0.1.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (158) hide show
  1. package/README.md +58 -125
  2. package/dist/agent/config.js +6 -15
  3. package/dist/agent/credentials.js +11 -23
  4. package/dist/agent/events.js +42 -64
  5. package/dist/agent/listing.js +39 -92
  6. package/dist/agent/pi.js +103 -252
  7. package/dist/boards/boards.js +19 -29
  8. package/dist/channels/attach.js +14 -42
  9. package/dist/channels/chains.js +33 -37
  10. package/dist/channels/chunk.js +8 -28
  11. package/dist/channels/commands.js +3 -14
  12. package/dist/channels/config.js +33 -52
  13. package/dist/channels/control.js +4 -13
  14. package/dist/channels/conversations.js +8 -25
  15. package/dist/channels/dedup.js +8 -17
  16. package/dist/channels/gatekeeper.js +13 -23
  17. package/dist/channels/lark-api.js +23 -63
  18. package/dist/channels/lark-outbound.js +12 -44
  19. package/dist/channels/lark-panel.js +7 -23
  20. package/dist/channels/lark-render.js +18 -62
  21. package/dist/channels/lark.js +52 -141
  22. package/dist/channels/lines.js +13 -15
  23. package/dist/channels/panel.js +16 -36
  24. package/dist/channels/receipts.js +29 -52
  25. package/dist/channels/routes.js +3 -9
  26. package/dist/channels/runtime.js +12 -23
  27. package/dist/channels/slack-api.js +34 -86
  28. package/dist/channels/slack-directory.js +7 -23
  29. package/dist/channels/slack-outbound.js +12 -56
  30. package/dist/channels/slack-panel.js +4 -13
  31. package/dist/channels/slack-render.js +23 -91
  32. package/dist/channels/slack-tool.js +48 -171
  33. package/dist/channels/slack.js +73 -239
  34. package/dist/channels/telegram-api.js +8 -20
  35. package/dist/channels/telegram-panel.js +5 -21
  36. package/dist/channels/telegram-render.js +13 -40
  37. package/dist/channels/telegram.js +54 -146
  38. package/dist/channels/types.js +5 -16
  39. package/dist/cli.js +17 -41
  40. package/dist/config-sync.js +87 -4
  41. package/dist/core/hub.js +7 -20
  42. package/dist/core/identity.js +20 -59
  43. package/dist/core/inbound-file.js +15 -49
  44. package/dist/core/inbox.js +12 -34
  45. package/dist/core/queue.js +3 -5
  46. package/dist/core/reply.js +41 -142
  47. package/dist/core/router.js +209 -264
  48. package/dist/core/types.js +4 -0
  49. package/dist/db.js +88 -272
  50. package/dist/drain.js +57 -50
  51. package/dist/extensions/index.js +3 -11
  52. package/dist/extensions/web/anthropic.js +3 -9
  53. package/dist/extensions/web/artifacts.js +2 -5
  54. package/dist/extensions/web/content.js +4 -12
  55. package/dist/extensions/web/http.js +2 -6
  56. package/dist/extensions/web/language.js +8 -18
  57. package/dist/extensions/web/openai.js +1 -1
  58. package/dist/extensions/web/provider.js +5 -18
  59. package/dist/extensions/web/tools.js +19 -63
  60. package/dist/lock.js +98 -0
  61. package/dist/log.js +9 -26
  62. package/dist/main.js +84 -183
  63. package/dist/paths.js +10 -26
  64. package/dist/secrets.js +18 -45
  65. package/dist/service.js +31 -73
  66. package/dist/settings.js +19 -63
  67. package/dist/tasks/agent.js +24 -45
  68. package/dist/tasks/callbacks.js +8 -16
  69. package/dist/tasks/command.js +2 -6
  70. package/dist/tasks/definitions.js +39 -62
  71. package/dist/tasks/execution.js +41 -39
  72. package/dist/tasks/groups.js +8 -11
  73. package/dist/tasks/messages.js +88 -155
  74. package/dist/tasks/outbox.js +33 -54
  75. package/dist/tasks/routes.js +4 -7
  76. package/dist/tasks/runs.js +4 -9
  77. package/dist/tasks/service.js +29 -42
  78. package/dist/tasks/store.js +36 -27
  79. package/dist/tasks/tool.js +59 -60
  80. package/dist/tools-task.js +20 -60
  81. package/dist/tools.js +98 -325
  82. package/dist/update.js +20 -43
  83. package/dist/web/auth.js +118 -179
  84. package/dist/web/config-sync.js +2 -2
  85. package/dist/web/config.js +3 -7
  86. package/dist/web/explorer.js +10 -21
  87. package/dist/web/fs.js +20 -42
  88. package/dist/web/instance.js +35 -82
  89. package/dist/web/providers.js +5 -11
  90. package/dist/web/public/assets/{activity-Bl3vZukb.js → activity-B89_hH7q.js} +1 -1
  91. package/dist/web/public/assets/activity-B89_hH7q.js.br +0 -0
  92. package/dist/web/public/assets/activity-B89_hH7q.js.gz +0 -0
  93. package/dist/web/public/assets/{boards-DYuf4Mlj.js → boards-BeKW0ZXK.js} +1 -1
  94. package/dist/web/public/assets/boards-BeKW0ZXK.js.br +0 -0
  95. package/dist/web/public/assets/boards-BeKW0ZXK.js.gz +0 -0
  96. package/dist/web/public/assets/explorer-DIuMlaV3.js +4 -0
  97. package/dist/web/public/assets/explorer-DIuMlaV3.js.br +0 -0
  98. package/dist/web/public/assets/explorer-DIuMlaV3.js.gz +0 -0
  99. package/dist/web/public/assets/index-DzXDXra_.js +85 -0
  100. package/dist/web/public/assets/index-DzXDXra_.js.br +0 -0
  101. package/dist/web/public/assets/index-DzXDXra_.js.gz +0 -0
  102. package/dist/web/public/assets/index-eqQLVS8Q.css +2 -0
  103. package/dist/web/public/assets/index-eqQLVS8Q.css.br +0 -0
  104. package/dist/web/public/assets/index-eqQLVS8Q.css.gz +0 -0
  105. package/dist/web/public/assets/{runs-BLJu7EXN.js → runs-Cwy0mN8i.js} +1 -1
  106. package/dist/web/public/assets/runs-Cwy0mN8i.js.br +0 -0
  107. package/dist/web/public/assets/runs-Cwy0mN8i.js.gz +0 -0
  108. package/dist/web/public/assets/{settings-BrdVh-Zi.js → settings-DzZLmujq.js} +1 -1
  109. package/dist/web/public/assets/settings-DzZLmujq.js.br +0 -0
  110. package/dist/web/public/assets/settings-DzZLmujq.js.gz +0 -0
  111. package/dist/web/public/assets/{task-runs-CeQS1rxa.js → task-runs-BCakxFk8.js} +1 -1
  112. package/dist/web/public/assets/task-runs-BCakxFk8.js.br +0 -0
  113. package/dist/web/public/assets/task-runs-BCakxFk8.js.gz +0 -0
  114. package/dist/web/public/assets/tasks-BlzEbk11.js +4 -0
  115. package/dist/web/public/assets/tasks-BlzEbk11.js.br +0 -0
  116. package/dist/web/public/assets/tasks-BlzEbk11.js.gz +0 -0
  117. package/dist/web/public/index.html +30 -16
  118. package/dist/web/public/index.html.br +0 -0
  119. package/dist/web/public/index.html.gz +0 -0
  120. package/dist/web/public/sw.js +14 -2
  121. package/dist/web/public/sw.js.br +0 -0
  122. package/dist/web/public/sw.js.gz +0 -0
  123. package/dist/web/push.js +55 -77
  124. package/dist/web/route.js +3 -7
  125. package/dist/web/server.js +109 -190
  126. package/dist/web/session-state.js +13 -53
  127. package/dist/web/types.js +2 -4
  128. package/dist/web/webpush.js +10 -25
  129. package/docs/deploy.md +115 -330
  130. package/package.json +1 -1
  131. package/skills/pier-boards/SKILL.md +81 -160
  132. package/skills/pier-help/SKILL.md +23 -20
  133. package/skills/pier-slack/SKILL.md +2 -2
  134. package/skills/pier-tasks/SKILL.md +23 -15
  135. package/dist/config-sync-fetch.js +0 -84
  136. package/dist/limits.js +0 -14
  137. package/dist/web/public/assets/activity-Bl3vZukb.js.br +0 -0
  138. package/dist/web/public/assets/activity-Bl3vZukb.js.gz +0 -0
  139. package/dist/web/public/assets/boards-DYuf4Mlj.js.br +0 -0
  140. package/dist/web/public/assets/boards-DYuf4Mlj.js.gz +0 -0
  141. package/dist/web/public/assets/explorer-qJH_9nTE.js +0 -4
  142. package/dist/web/public/assets/explorer-qJH_9nTE.js.br +0 -0
  143. package/dist/web/public/assets/explorer-qJH_9nTE.js.gz +0 -0
  144. package/dist/web/public/assets/index-Dqdb-Eqt.js +0 -85
  145. package/dist/web/public/assets/index-Dqdb-Eqt.js.br +0 -0
  146. package/dist/web/public/assets/index-Dqdb-Eqt.js.gz +0 -0
  147. package/dist/web/public/assets/index-DzmMzvi_.css +0 -2
  148. package/dist/web/public/assets/index-DzmMzvi_.css.br +0 -0
  149. package/dist/web/public/assets/index-DzmMzvi_.css.gz +0 -0
  150. package/dist/web/public/assets/runs-BLJu7EXN.js.br +0 -0
  151. package/dist/web/public/assets/runs-BLJu7EXN.js.gz +0 -0
  152. package/dist/web/public/assets/settings-BrdVh-Zi.js.br +0 -0
  153. package/dist/web/public/assets/settings-BrdVh-Zi.js.gz +0 -0
  154. package/dist/web/public/assets/task-runs-CeQS1rxa.js.br +0 -0
  155. package/dist/web/public/assets/task-runs-CeQS1rxa.js.gz +0 -0
  156. package/dist/web/public/assets/tasks-bcb3fYdK.js +0 -4
  157. package/dist/web/public/assets/tasks-bcb3fYdK.js.br +0 -0
  158. package/dist/web/public/assets/tasks-bcb3fYdK.js.gz +0 -0
package/dist/db.js CHANGED
@@ -1,44 +1,25 @@
1
- // The one connection, and the one place the schema is written down.
2
- //
3
- // Every store used to open `pier.db` for itself and create its own tables with
4
- // `CREATE TABLE IF NOT EXISTS`. That works exactly once: it can add a table but
5
- // never change one, so the first column an upgrade needed would have left every
6
- // existing instance with a schema nothing could repair. `user_version` is a
7
- // single number per *database*, not per table, which is why the schema cannot
8
- // stay spread across five modules — and five connections to one file is also
9
- // five writers competing for the same lock.
10
- //
11
- // So: one connection, one ordered list of migrations, applied in one
12
- // transaction before any store exists. A store receives the handle and owns
13
- // only its queries.
1
+ // The one connection, and the one place the schema is written down: one ordered
2
+ // list of migrations, applied in one transaction before any store exists.
3
+ // `user_version` is per database, not per table, so the schema cannot be
4
+ // spread across modules.
14
5
  import { chmodSync, existsSync, mkdirSync, readdirSync, renameSync, rmSync, statSync } from "node:fs";
15
6
  import { basename, dirname, join } from "node:path";
16
7
  import { DatabaseSync } from "node:sqlite";
17
8
  import { logger } from "./log.js";
18
9
  import { PIER_DB } from "./paths.js";
19
10
  const log = logger("db");
20
- /** Snapshots to keep *of each kind*. Three is two upgrades of regret plus one:
21
- * they are full copies of the database, and the one that matters is the
22
- * newest. Counted per kind because the two kinds answer different questions —
23
- * a run of releases must not evict the pre-migration copies. */
11
+ /** Per kind, because full copies are large and a run of releases must not
12
+ * evict the pre-migration copies. */
24
13
  const KEEP_BACKUPS = 3;
25
- /** How long a second process may wait for the write lock before failing. Two
26
- * Pier processes on one PIER_HOME contend exactly once — at boot, when both
27
- * want to migrate — and failing instantly there turns a restart race into a
28
- * crash loop. */
14
+ /** Two Pier processes on one PIER_HOME contend at boot, when both want to
15
+ * migrate; failing instantly turns a restart race into a crash loop. */
29
16
  const BUSY_TIMEOUT_MS = 5_000;
30
- /**
31
- * Append-only, never edited: index + 1 is the `user_version` a database is at
32
- * once that entry has run. An entry that shipped is history — fix a mistake
33
- * with the next one, because somebody's database already ran the old one.
34
- *
35
- * Migration 1 is the whole schema as of 0.0.1 and assumes nothing before it:
36
- * pre-release databases are not upgraded, they are deleted.
37
- */
17
+ /** Append-only, never edited: index + 1 is the `user_version` after that entry.
18
+ * Fix a mistake with the next one — somebody's database already ran the old one. */
38
19
  const MIGRATIONS = [
39
20
  // 1 — the 0.0.1 schema.
40
21
  `
41
- -- The single credential in front of every HTTP surface (web/auth.ts).
22
+ -- The single credential in front of every HTTP surface.
42
23
  CREATE TABLE auth (
43
24
  id INTEGER PRIMARY KEY CHECK (id = 1),
44
25
  salt TEXT NOT NULL,
@@ -46,15 +27,13 @@ const MIGRATIONS = [
46
27
  created_at INTEGER NOT NULL
47
28
  );
48
29
 
49
- -- Instance facts that are neither a credential nor per-session; one row per
50
- -- setting, so the next setting is not the next table.
30
+ -- Instance facts that are neither a credential nor per-session.
51
31
  CREATE TABLE settings (
52
32
  key TEXT PRIMARY KEY,
53
33
  value TEXT NOT NULL
54
34
  );
55
35
 
56
- -- Workbench bookkeeping: pinned = listed under Projects, unread = a turn
57
- -- finished that no client has acknowledged.
36
+ -- Workbench bookkeeping; unread = a turn finished that no client acknowledged.
58
37
  CREATE TABLE session_state (
59
38
  session_id TEXT PRIMARY KEY,
60
39
  pinned INTEGER NOT NULL DEFAULT 0,
@@ -87,8 +66,8 @@ const MIGRATIONS = [
87
66
  PRIMARY KEY (platform, chat_id, message_id)
88
67
  );
89
68
 
90
- -- Scheduled work. The row keeps its whole JSON document; the columns beside
91
- -- it are only what a query filters or orders by.
69
+ -- Scheduled work: the JSON is the document, the columns beside it are only
70
+ -- what a query filters or orders by.
92
71
  CREATE TABLE tasks (
93
72
  id TEXT PRIMARY KEY,
94
73
  updated_at INTEGER NOT NULL,
@@ -119,10 +98,9 @@ const MIGRATIONS = [
119
98
  json TEXT NOT NULL
120
99
  );
121
100
  `,
122
- // 2 — provider credentials move from <agentDir>/auth.json into the database.
101
+ // 2 — provider credentials.
123
102
  `
124
103
  -- One row per provider (key = provider id), value sealed by secrets.ts.
125
- -- Owned by agent/credentials.ts.
126
104
  CREATE TABLE credentials (
127
105
  key TEXT PRIMARY KEY,
128
106
  value TEXT NOT NULL
@@ -130,8 +108,7 @@ const MIGRATIONS = [
130
108
  `,
131
109
  // 3 — what a restart's drain deadline cut off, told to the chat at next boot.
132
110
  `
133
- -- Written only when a graceful restart aborts a still-running turn; the next
134
- -- boot delivers each row and clears it once delivered. Owned by drain.ts.
111
+ -- Turns a graceful restart cut off; each row is delivered at next boot, then cleared.
135
112
  CREATE TABLE restart_ledger (
136
113
  id INTEGER PRIMARY KEY,
137
114
  channel_id TEXT NOT NULL,
@@ -146,10 +123,9 @@ const MIGRATIONS = [
146
123
  ALTER TABLE session_state ADD COLUMN title TEXT;
147
124
  ALTER TABLE session_state ADD COLUMN created_at INTEGER;
148
125
  `,
149
- // 5 — the workbench can reach a browser that is not open (web/push.ts).
126
+ // 5 — Web Push.
150
127
  `
151
- -- One row per browser that asked to be notified, exactly as the Push API
152
- -- described it; a dead endpoint is deleted when its service says so.
128
+ -- One row per browser that asked to be notified, as the Push API described it.
153
129
  CREATE TABLE push_subscriptions (
154
130
  endpoint TEXT PRIMARY KEY,
155
131
  p256dh TEXT NOT NULL,
@@ -158,8 +134,8 @@ const MIGRATIONS = [
158
134
  created_at INTEGER NOT NULL
159
135
  );
160
136
 
161
- -- This instance's VAPID identity: one key pair, minted on first use. Every
162
- -- subscription above is bound to it, so it is never rotated on its own.
137
+ -- This instance's VAPID key pair; every subscription is bound to it, so it
138
+ -- is never rotated on its own.
163
139
  CREATE TABLE push_identity (
164
140
  id INTEGER PRIMARY KEY CHECK (id = 1),
165
141
  public_key TEXT NOT NULL,
@@ -167,21 +143,16 @@ const MIGRATIONS = [
167
143
  created_at INTEGER NOT NULL
168
144
  );
169
145
  `,
170
- // 6 — Projects keeps the order the workbench was put in, by hand.
146
+ // 6 — manual order in the rail.
171
147
  `
172
- -- Manual order, both nullable: a row nobody has dragged sorts on top of the
173
- -- list it belongs to, so a fresh database needs no backfill. sort places a
174
- -- session inside its project; project_sort places the project, carried on
175
- -- every one of its rows because a project is a cwd, not a table.
148
+ -- NULL sorts on top, so a fresh database needs no backfill.
176
149
  ALTER TABLE session_state ADD COLUMN sort INTEGER;
177
150
  ALTER TABLE session_state ADD COLUMN project_sort INTEGER;
178
151
  `,
179
- // 7 — the session listing, so a transcript is read once (agent/listing.ts).
152
+ // 7 — the session listing, so a transcript is read once.
180
153
  `
181
- -- One row per session file. (size, mtime) is what makes the row usable
182
- -- without opening the file; parsed_bytes is where reading resumes when it
183
- -- grew, and is always a line boundary. Derived from disk and disposable: a
184
- -- deleted row costs one re-read, never a fact.
154
+ -- One row per session file, derived from disk and disposable. A row whose
155
+ -- (size, mtime) match is trusted unopened; parsed_bytes is where reading resumes, always a line boundary.
185
156
  CREATE TABLE session_index (
186
157
  path TEXT PRIMARY KEY,
187
158
  id TEXT NOT NULL,
@@ -194,73 +165,39 @@ const MIGRATIONS = [
194
165
  parsed_bytes INTEGER NOT NULL
195
166
  );
196
167
  `,
197
- // 8 — Projects holds a working set: what is warm, plus what is kept.
168
+ // 8 — working-set lease columns (dropped again in 9 and 11).
198
169
  `
199
- -- Membership was permanent, so every throwaway session stayed in the rail
200
- -- until someone removed it by hand. last_active is the lease: the end of a
201
- -- turn renews it, and web/session-state.ts stops listing a row that ran out.
202
- -- kept opts one row out of expiry entirely — what the pin control now means.
203
170
  ALTER TABLE session_state ADD COLUMN kept INTEGER NOT NULL DEFAULT 0;
204
171
  ALTER TABLE session_state ADD COLUMN last_active INTEGER;
205
- -- Left NULL on purpose: the honest value is when the transcript was last
206
- -- written, which only a listing knows. web/server.ts pays one for a database
207
- -- carrying rows without it, the same gate the pin backfill already uses, so
208
- -- the first rail after an upgrade is dated by use and not by creation.
209
172
  `,
210
- // 9 — the summary a transcript already carries is read, not mirrored.
173
+ // 9 — drop the columns that mirrored the transcript.
211
174
  `
212
- -- Dropped rather than left unread: a column nobody writes still answers when
213
- -- somebody selects it, and the next reader has no way to tell a stale title
214
- -- from a current one. The pre-migration backup beside the database is the
215
- -- way back, not a row of fossils. cwd stays — it is the key a project's
216
- -- manual place is stamped on, and it never changes for a session.
217
175
  ALTER TABLE session_state DROP COLUMN title;
218
176
  ALTER TABLE session_state DROP COLUMN created_at;
219
177
  ALTER TABLE session_state DROP COLUMN last_active;
220
178
  `,
221
- // 10 — taking a session into Projects is itself an act, and it is dated.
179
+ // 10 — pinned_at (dropped again in 11).
222
180
  `
223
- -- When a hand last put this session in Projects (pin, or a keep toggle).
224
- -- Not the mirror migration 9 removed: last_active was a copy of a fact the
225
- -- transcript owns, while this one exists nowhere else — pinning a cold
226
- -- session back is a statement that it is warm again, and without a record of
227
- -- *when* it was made the row is dropped by the same read that drew it.
228
- -- NULL for every row that predates this: never pinned within a lease.
229
181
  ALTER TABLE session_state ADD COLUMN pinned_at INTEGER;
230
182
  `,
231
- // 11 — Projects holds what a hand put there, for as long as the hand says.
183
+ // 11 — the lease is gone, so both of its columns are.
232
184
  `
233
- -- The lease is gone, so both of its columns are. It expired nothing: a row
234
- -- it dropped kept its transcript, its place and its ownership, and one more
235
- -- turn brought it back — so what it actually did was hide rows nobody asked
236
- -- it to hide, and kept existed only to opt out of that. On the instance this
237
- -- was decided on, the lease had never dropped a row: 20 pinned sessions,
238
- -- none past seven days, one kept. Removing a row from Projects is the ✓ on
239
- -- the row, and it stays the only way out.
240
185
  ALTER TABLE session_state DROP COLUMN kept;
241
186
  ALTER TABLE session_state DROP COLUMN pinned_at;
242
187
  `,
243
- // 12 — one row is how two processes take turns (src/tools.ts).
188
+ // 12 — the cross-process tools sync lock.
244
189
  `
245
- -- The tools sync, held across processes: the token says who holds it, the
246
- -- heartbeat says they are still alive. Both processes already open this
247
- -- database, and BEGIN IMMEDIATE is real mutual exclusion — a lock file with
248
- -- a pid in it is neither, which is what this replaces. One row, because
249
- -- there is one thing to serialize; the second lock can bring its own table
250
- -- and its own reason for existing.
190
+ -- One row: token says who holds the lock, heartbeat_at says they are still alive.
251
191
  CREATE TABLE tools_sync_lock (
252
192
  id INTEGER PRIMARY KEY CHECK (id = 1),
253
193
  token TEXT NOT NULL,
254
194
  heartbeat_at INTEGER NOT NULL
255
195
  );
256
196
  `,
257
- // 13 — a signed-in browser can be signed out on its own (web/auth.ts).
197
+ // 13 — web sessions.
258
198
  `
259
- -- One row per signed-in browser. The cookie carries "<id>.<token>" and only
260
- -- the token's SHA-256 is stored, so a copy of this database cannot be turned
261
- -- into a session — and deleting a row is what revocation is. seen_at is the
262
- -- whole lifetime: the session ends one TTL after it, so there is no second
263
- -- column that can disagree about when.
199
+ -- One row per signed-in browser; only the cookie token's SHA-256 is stored,
200
+ -- deleting a row is revocation, and the session ends one TTL after seen_at.
264
201
  CREATE TABLE web_sessions (
265
202
  id TEXT PRIMARY KEY,
266
203
  token_hash TEXT NOT NULL,
@@ -270,15 +207,9 @@ const MIGRATIONS = [
270
207
  agent TEXT NOT NULL
271
208
  );
272
209
  `,
273
- // 14 — signing a browser out also stops notifying it (web/push.ts).
210
+ // 14 — a push subscription belongs to the web session that made it.
274
211
  `
275
- -- A subscription belongs to the web session that made it, and dies with it:
276
- -- the cascade is the rule, so no code has to remember to run it — revoking a
277
- -- session, changing the password and recovering it all reach here for free.
278
- -- Rebuilt rather than altered because a foreign key cannot be added to an
279
- -- existing table; nothing is carried over, since migration 13 invalidated
280
- -- every cookie and each of these rows belongs to a browser that is now
281
- -- signed out. A browser re-subscribes on its next load.
212
+ -- Rebuilt: a foreign key cannot be added to an existing table.
282
213
  DROP TABLE push_subscriptions;
283
214
  CREATE TABLE push_subscriptions (
284
215
  endpoint TEXT PRIMARY KEY,
@@ -289,23 +220,14 @@ const MIGRATIONS = [
289
220
  session_id TEXT NOT NULL REFERENCES web_sessions(id) ON DELETE CASCADE
290
221
  );
291
222
  `,
292
- // 15 — the scheduler's tick stops reading the rows it cannot deliver.
223
+ // 15 — indexes for the scheduler's once-a-second sweep.
293
224
  `
294
- -- Once a second, tasks/ asks for the runs whose callback is still owed and
295
- -- the messages whose injection has not landed. Both are a handful of rows
296
- -- filtered on one low-cardinality column, and without an index both are a
297
- -- full scan of a table that only grows — the sweep got slower with every
298
- -- run that finished cleanly and can never match again.
299
225
  CREATE INDEX task_runs_callback_state ON task_runs(callback_state);
300
226
  CREATE INDEX task_messages_state ON task_messages(state);
301
227
  `,
302
- // 16 — the same tick stops parsing every task document to find none due.
228
+ // 16 — an indexable next-due value, with the JSON still the only record.
303
229
  `
304
- -- When a task is next due is the one field the scheduler asks about once a
305
- -- second, and it lived only inside the JSON: answering meant reading and
306
- -- parsing every definition, due or not. A generated column keeps the JSON
307
- -- as the only record while giving the query planner a value it can index; a
308
- -- disabled or archived task has no next run, so NULLs stay out of the index.
230
+ -- A disabled or archived task has no next run, so NULLs stay out of the index.
309
231
  ALTER TABLE tasks ADD COLUMN next_run_at INTEGER
310
232
  GENERATED ALWAYS AS (json_extract(json, '$.nextRunAt')) VIRTUAL;
311
233
  CREATE INDEX tasks_due ON tasks(next_run_at) WHERE next_run_at IS NOT NULL;
@@ -316,79 +238,43 @@ const MIGRATIONS = [
316
238
  CREATE INDEX task_runs_visible_time ON task_runs(queued_at DESC, id DESC)
317
239
  WHERE NOT (state = 'succeeded' AND json_extract(json, '$.matched') IS 0);
318
240
  `,
319
- // 18 — opening a session lists every run it delegated, not the last hour's.
241
+ // 18 — runs by delegating session.
320
242
  `
321
- -- The run cards are the messages a session sent, so the transcript wants
322
- -- all of them: the query is by the delegating session, which lived only in
323
- -- the JSON — without this every session open was a full scan of task_runs.
324
243
  CREATE INDEX task_runs_invoked_by ON task_runs(json_extract(json, '$.invokedBySessionId'), queued_at DESC);
325
244
  `,
326
- // 19 — the rail is one flat list, and pinned means "stuck to the top".
245
+ // 19 — pinned now means "stuck to the top"; nobody put a historical session there.
327
246
  `
328
- -- pinned used to mean "listed under Projects", and every session created in
329
- -- the workbench was. Now it means on top of the list, and nobody put a
330
- -- historical session there: kept as-is, every web session ever made would
331
- -- land in the pinned section. project_sort stays as a column nothing reads
332
- -- — a SQLite column drop rewrites the table for a NULL nobody pays for.
333
247
  UPDATE session_state SET pinned = 0;
334
248
  `,
335
- // 20 — the top of the rail is maintained, not arranged: no pin, no drag.
249
+ // 20 — sort becomes the rank in the working set the rail keeps on top; pinned goes.
336
250
  `
337
- -- sort was the place a hand dragged a pinned row to; it is now the rank in
338
- -- the working set the rail keeps on top, which a session enters by being
339
- -- spoken to and leaves by being pushed out of the last slot. The pinned rows
340
- -- are that set's first members — they are what somebody was working on — in
341
- -- the order they were arranged in; never-dragged ones sorted first, so -1 is
342
- -- the rank that keeps them there.
251
+ -- Pinned rows seed the set at rank -1, capped at the set's size (8).
343
252
  UPDATE session_state SET sort = -1 WHERE pinned = 1 AND sort IS NULL;
344
253
  UPDATE session_state SET sort = NULL WHERE pinned = 0;
345
- -- The set has a size (web/session-state.ts); more pins than that is a list
346
- -- the promotion rule would never have built.
347
254
  UPDATE session_state SET sort = NULL WHERE session_id IN (
348
255
  SELECT session_id FROM session_state WHERE sort IS NOT NULL
349
256
  ORDER BY sort, session_id LIMIT -1 OFFSET 8
350
257
  );
351
258
  ALTER TABLE session_state DROP COLUMN pinned;
352
259
  `,
353
- // 21 — unread is the workbench's own attention, so it is only its own rows.
260
+ // 21 — unread is only written for web sessions now; clear the marks nobody could ack.
354
261
  `
355
- -- The flag was written for every session whose turn ended, including the
356
- -- ones no browser is the reader of: an IM session answers its chat, a run's
357
- -- session answers its supervisor. Both are now skipped at the write
358
- -- (web/server.ts), and neither could ever be acked — that needs the session
359
- -- on screen — so the rows they left would stay set forever. On the instance
360
- -- this was decided on, 195 of 196 marks were those. Cleared wholesale rather
361
- -- than by owner: the one real row is a turn from before an upgrade nobody
362
- -- was watching for, and a false amber dot costs less than the join.
363
262
  UPDATE session_state SET unread = 0;
364
263
  `,
365
- // 22 — the palette finds what was said, not only what a session is called.
264
+ // 22 — message search.
366
265
  `
367
- -- One row per user message and per assistant reply, keyed by the transcript
368
- -- it came from so a file that is rewritten or gone drops its rows in one
369
- -- statement (agent/listing.ts). Trigram tokens: a substring match with no
370
- -- word segmentation, which CJK text has none of and a path or an identifier
371
- -- in a prompt has too much of. Steps stay out — tool calls, their output,
372
- -- thinking are the bulk of a transcript and nobody searches for what ran.
266
+ -- One row per user message and assistant reply, keyed by transcript path.
267
+ -- Trigram: substring match with no word segmentation, which CJK text has none of.
373
268
  CREATE VIRTUAL TABLE session_fts USING fts5(
374
269
  text, session_id UNINDEXED, path UNINDEXED, role UNINDEXED, at UNINDEXED,
375
270
  tokenize = 'trigram'
376
271
  );
377
- -- Derived and disposable (migration 7): every row goes, so the next scan
378
- -- reads every transcript from its first byte and fills the table above.
379
- -- Resetting parsed_bytes alone would not — a row whose (size, mtime) still
380
- -- match is trusted without opening the file.
272
+ -- Every row goes so the next scan re-reads each transcript and fills the table above.
381
273
  DELETE FROM session_index;
382
274
  `,
383
275
  ];
384
- /**
385
- * Several writes as one, or none. `BEGIN IMMEDIATE` because every writer here
386
- * competes with another Pier process on the same file: taking the write lock
387
- * up front turns a race into a wait, where deferred would turn it into
388
- * SQLITE_BUSY halfway through. The rollback is the reason this is shared —
389
- * three modules had written the same seven lines, and a `catch` that forgets
390
- * to roll back leaves the connection in a transaction forever.
391
- */
276
+ /** `BEGIN IMMEDIATE`: taking the write lock up front turns a race with another
277
+ * Pier process into a wait, where deferred would be SQLITE_BUSY halfway through. */
392
278
  export function transact(db, work) {
393
279
  db.exec("BEGIN IMMEDIATE");
394
280
  try {
@@ -401,15 +287,8 @@ export function transact(db, work) {
401
287
  throw err;
402
288
  }
403
289
  }
404
- /**
405
- * Prepared statements, memoized by their SQL. `prepare()` compiles, and the
406
- * callers here hand it the same handful of strings forever — twice per
407
- * authenticated request, once a second per scheduler sweep. A `StatementSync`
408
- * is reusable with different bound parameters, so one per SQL string per
409
- * connection is the whole cache. Bound to the connection because a statement
410
- * belongs to the database that compiled it; SQL built per call does not
411
- * belong in here, since the cache would then grow without a bound.
412
- */
290
+ /** Prepared statements memoized by SQL; callers hand it the same handful of
291
+ * strings forever. SQL built per call does not belong here: the cache is unbounded. */
413
292
  export function statements(db) {
414
293
  const cache = new Map();
415
294
  return (sql) => {
@@ -420,28 +299,15 @@ export function statements(db) {
420
299
  };
421
300
  }
422
301
  let shared;
423
- /**
424
- * The process's one connection, opened and migrated on first use. Every store
425
- * defaults to it; a test passes `openDb(":memory:")` instead.
426
- */
302
+ /** The process's one connection; a test passes `openDb(":memory:")` instead. */
427
303
  export const pierDb = () => (shared ??= openDb(PIER_DB));
428
- /** A release-level restore point, taken for every release even when it has no
429
- * schema migration. The previous complete copies stay put if writing this one
430
- * fails, and the service may be running while it is written: `copyDatabase`
431
- * reads through a read-only connection, so what it writes is one consistent
432
- * snapshot of a live database rather than a torn `cp`.
433
- *
434
- * `version` is the Pier that produced this database, not the one being
435
- * installed: the updater runs this from the tree it is about to replace, and
436
- * restoring a database means reinstalling the code that speaks its schema
437
- * (`migrate` refuses one from a newer Pier). So the name carries the other half
438
- * of the pair. Backing up twice at one version replaces that version's copy —
439
- * the pairing is identical, so a second name for it would say nothing. */
304
+ /** A restore point per release, schema migration or not. `version` is the Pier
305
+ * that produced the database: restoring one means reinstalling the code that
306
+ * speaks its schema, so the name carries that half of the pair. */
440
307
  export function backupDb(version, path = PIER_DB) {
441
308
  if (!existsSync(path))
442
309
  return undefined;
443
- // In a filename, so it may not carry a separator or a traversal; a version
444
- // this malformed is a broken install, not something to guess at.
310
+ // In a filename, so no separator or traversal.
445
311
  const safe = version.replaceAll(/[^0-9A-Za-z.+-]/g, "_") || "unknown";
446
312
  const bak = join(backupsDir(path, true), `${basename(path)}.release-${safe}.bak`);
447
313
  copyDatabase(path, bak);
@@ -449,67 +315,49 @@ export function backupDb(version, path = PIER_DB) {
449
315
  prune(releases(path));
450
316
  return bak;
451
317
  }
452
- /** Open a database, bring it to the current schema, and lock down its files.
453
- * `migrations` is injectable only so tests can exercise an upgrade — there is
454
- * exactly one real list. */
318
+ /** `migrations` is injectable only so tests can exercise an upgrade. */
455
319
  export function openDb(path, migrations = MIGRATIONS) {
456
320
  if (path !== ":memory:")
457
321
  mkdirSync(dirname(path), { recursive: true, mode: 0o700 });
458
322
  const db = new DatabaseSync(path);
459
323
  // Timeout first: two processes booting together contend on the WAL switch
460
- // itself. Outside the transaction below: journal_mode is a property of the
461
- // file, and SQLite refuses to change it inside one.
324
+ // itself, and journal_mode cannot change inside a transaction.
462
325
  db.exec(`PRAGMA busy_timeout = ${BUSY_TIMEOUT_MS}`);
463
326
  db.exec("PRAGMA journal_mode = WAL");
464
- // WAL's default leaves every commit waiting on an fsync, and DatabaseSync is
465
- // synchronous — that wait is the event loop's. NORMAL still survives a
466
- // process crash; only a power loss can cost the last transactions, which for
467
- // routing state and task bookkeeping is a fair trade for not blocking.
327
+ // DatabaseSync is synchronous, so WAL's per-commit fsync would be the event
328
+ // loop's wait. NORMAL survives a process crash; only power loss costs anything.
468
329
  db.exec("PRAGMA synchronous = NORMAL");
469
- // Off by default in SQLite, and a declared relationship nothing enforces is
470
- // a comment. Set before migrate(): it is a per-connection switch and a no-op
471
- // inside a transaction. Nothing older declares a key, so this changes the
472
- // behaviour of exactly one table — push_subscriptions, whose rows must not
473
- // outlive the session that made them.
330
+ // Off by default in SQLite; per connection, and a no-op inside a transaction.
474
331
  db.exec("PRAGMA foreign_keys = ON");
475
332
  migrate(db, path, migrations);
476
333
  if (path !== ":memory:")
477
334
  restrict(path);
478
335
  return db;
479
336
  }
480
- /**
481
- * Upgrades only. `user_version` counts up and nothing counts it back down, so a
482
- * database from a newer Pier is refused rather than served: the old code would
483
- * happily write the new schema's tables and lose whatever it did not know
484
- * about. The way back is the `.bak` this function writes before upgrading.
485
- */
337
+ /** Upgrades only: a database from a newer Pier is refused, since old code would
338
+ * write the new schema's tables and lose what it did not know about. */
486
339
  function migrate(db, path, migrations) {
487
340
  const { user_version: at } = db.prepare("PRAGMA user_version").get();
488
341
  const target = migrations.length;
489
342
  if (at > target) {
490
- // Name the snapshot that exists rather than a pattern: the operator is
491
- // reading this because the service will not start.
343
+ // Name the snapshot that exists: the operator is reading this because the
344
+ // service will not start.
492
345
  const newest = path === ":memory:" ? undefined : snapshots(path)[0]?.file;
493
346
  throw new Error(`${path} is at schema ${at}, this Pier speaks ${target}: a database is ` +
494
347
  `never downgraded. Restore ${newest ?? `a copy from ${backupsDir(path)}`}, or run the newer Pier.`);
495
348
  }
496
- // Version 0 with tables is a database from before versioning existed.
497
- // Migration 1 assumes an empty file, so the collision it would hit says
498
- // "table already exists" — this says what is actually wrong and what to do.
349
+ // Version 0 with tables predates versioning; migration 1 assumes an empty file.
499
350
  if (at === 0 && db.prepare("SELECT 1 FROM sqlite_master WHERE type = 'table' LIMIT 1").get()) {
500
351
  throw new Error(`${path} predates schema versioning and cannot be upgraded — nothing was changed. ` +
501
352
  `Move the file aside and restart; channel credentials, tasks and the password start over.`);
502
353
  }
503
354
  if (at === target)
504
355
  return;
505
- // One transaction for the statements *and* the version number: a crash
506
- // between them would leave a database whose version describes a schema it
507
- // does not have, which is worse than a crash.
356
+ // Statements and version number in one transaction: a version describing a
357
+ // schema the database does not have is worse than a crash.
508
358
  db.exec("BEGIN IMMEDIATE");
509
- // Re-read inside the lock. Two processes starting together both saw work to
510
- // do; the one that waited for the lock would otherwise replay migrations the
511
- // winner already committed and die on "table already exists", with a healthy
512
- // database in front of it.
359
+ // Re-read inside the lock: the process that waited must not replay what the
360
+ // winner already committed.
513
361
  const { user_version: locked } = db.prepare("PRAGMA user_version").get();
514
362
  if (locked >= target) {
515
363
  db.exec("ROLLBACK");
@@ -519,9 +367,8 @@ function migrate(db, path, migrations) {
519
367
  log.info(`schema already at ${locked}, migrated by another process`);
520
368
  return;
521
369
  }
522
- // Keep the write lock while a second, read-only connection copies the last
523
- // committed state. That connection may VACUUM while this one holds a RESERVED
524
- // lock; other Pier starts wait here instead of racing on the shared .tmp.
370
+ // Under the write lock: a read-only connection may VACUUM while this one
371
+ // holds RESERVED, and other Pier starts wait instead of racing on the .tmp.
525
372
  if (locked > 0 && path !== ":memory:") {
526
373
  try {
527
374
  snapshot(path, locked);
@@ -540,32 +387,22 @@ function migrate(db, path, migrations) {
540
387
  }
541
388
  catch (err) {
542
389
  db.exec("ROLLBACK");
543
- // Which one, and that the database is untouched: the operator's next move
544
- // is to restore a backup or to report the migration, and a bare SQLite
545
- // error says neither.
390
+ // Which one, and that the database is untouched: a bare SQLite error says neither.
546
391
  throw new Error(`migration ${step + 1} failed on ${path} — nothing was changed: ${String(err)}`, { cause: err });
547
392
  }
548
393
  log.info(locked === 0 ? `schema created at version ${target}` : `schema ${locked} → ${target}`);
549
394
  if (path !== ":memory:")
550
395
  prune(snapshots(path).map(({ file }) => file));
551
396
  }
552
- /**
553
- * The copy that exists because `user_version` only counts up: the transaction
554
- * above protects against a migration that *failed*, and this against one that
555
- * succeeded and should not have. `VACUUM INTO`, not `cp`: under WAL the
556
- * committed tail of the database lives in the `-wal` sidecar.
557
- *
558
- * Written under a temporary name and renamed into place. `VACUUM INTO` refuses
559
- * an existing target, so the alternative is deleting the previous snapshot
560
- * first — which means the likely failure here, a full disk, leaves neither the
561
- * old snapshot nor a complete new one. A rename is atomic: the `.bak` name only
562
- * ever refers to a finished copy.
563
- */
397
+ /** The transaction protects against a migration that failed; this against one
398
+ * that succeeded and should not have. */
564
399
  function snapshot(path, at) {
565
400
  const bak = join(backupsDir(path, true), `${basename(path)}.v${at}.bak`);
566
401
  copyDatabase(path, bak);
567
402
  log.info(`pre-migration backup: ${bak}`);
568
403
  }
404
+ /** `VACUUM INTO`, not `cp`: under WAL the committed tail lives in the `-wal`
405
+ * sidecar. Temp name plus rename, so a full disk leaves the old copy intact. */
569
406
  function copyDatabase(path, bak) {
570
407
  const tmp = `${bak}.tmp`;
571
408
  rmSync(tmp, { force: true }); // a previous crash may have left one
@@ -579,16 +416,9 @@ function copyDatabase(path, bak) {
579
416
  chmodSync(tmp, 0o600); // it holds everything the 0600 database holds
580
417
  renameSync(tmp, bak);
581
418
  }
582
- /**
583
- * One directory for every copy of this database, `db/backups/`. Beside the
584
- * database was fine while there was one snapshot per schema; a restore point
585
- * per release turns that into a listing where the live file and its sidecars
586
- * are hard to pick out, and "which of these do I not delete" is the wrong
587
- * question to make an operator answer under pressure.
588
- *
589
- * `create` also adopts what an older Pier wrote next to the database, so the
590
- * restore procedure names one location instead of two forever.
591
- */
419
+ /** `db/backups/`, so the live file and its sidecars are never in the listing an
420
+ * operator prunes under pressure. `create` adopts copies an older Pier left
421
+ * beside the database. */
592
422
  function backupsDir(path, create = false) {
593
423
  const dir = join(dirname(path), "backups");
594
424
  if (!create)
@@ -603,8 +433,7 @@ function backupsDir(path, create = false) {
603
433
  }
604
434
  return dir;
605
435
  }
606
- /** The copies of one kind: `v<schema>` or `release-<version>`. Disjoint
607
- * prefixes, so each kind is counted and pruned on its own. */
436
+ /** `v<schema>` or `release-<version>`: disjoint prefixes, pruned per kind. */
608
437
  function listBackups(path, kind) {
609
438
  const dir = backupsDir(path);
610
439
  if (!existsSync(dir))
@@ -612,8 +441,7 @@ function listBackups(path, kind) {
612
441
  const prefix = `${basename(path)}.${kind}`;
613
442
  return readdirSync(dir).filter((name) => name.startsWith(prefix) && name.endsWith(".bak"));
614
443
  }
615
- /** Pre-migration snapshots, newest schema first — the number in the name is an
616
- * ordinal, so it orders them without asking the filesystem. */
444
+ /** Pre-migration snapshots, newest schema first. */
617
445
  function snapshots(path) {
618
446
  const prefix = `${basename(path)}.v`;
619
447
  return listBackups(path, "v")
@@ -624,39 +452,27 @@ function snapshots(path) {
624
452
  .filter(({ version }) => Number.isInteger(version))
625
453
  .sort((a, b) => b.version - a.version);
626
454
  }
627
- /** Release restore points, newest copy first. Ordered by mtime: the name holds
628
- * a Pier version, and comparing those means reimplementing semver here — while
629
- * two updates of one instance are never in flight at the same moment. Legacy
630
- * `pier.db.release.bak` shares the prefix, so it ages out like the rest. */
455
+ /** Release restore points, newest first by mtime: ordering by the version in
456
+ * the name would mean reimplementing semver here. */
631
457
  function releases(path) {
632
458
  return listBackups(path, "release")
633
459
  .map((name) => join(backupsDir(path), name))
634
460
  .sort((a, b) => statSync(b).mtimeMs - statSync(a).mtimeMs);
635
461
  }
636
- /** Keep the newest few, oldest first out. Nobody restores a database from four
637
- * upgrades ago, and every one of these is the size of the whole database. */
638
462
  function prune(newestFirst) {
639
463
  for (const file of newestFirst.slice(KEEP_BACKUPS)) {
640
464
  rmSync(file, { force: true });
641
465
  log.info(`removed superseded backup: ${file}`);
642
466
  }
643
467
  }
644
- /**
645
- * The database holds the password hash, so it is not world-readable — and
646
- * neither are the sidecars, where a 0644 `-wal` would leak exactly what the
647
- * 0600 database is hiding. Done after the migration, so the sidecars that
648
- * writing created exist by now; SQLite gives later ones the database's mode.
649
- * The directory too: it exists only to hold this database and its sidecars
650
- * (paths.ts puts them under their own `db/`, away from the boards PIER_HOME
651
- * also holds), so nothing else needs to see into it.
652
- */
468
+ /** The database holds the password hash; a 0644 `-wal` would leak what the
469
+ * 0600 database hides. After the migration, so the sidecars exist by now. */
653
470
  function restrict(path) {
654
471
  for (const file of [path, `${path}-wal`, `${path}-shm`]) {
655
472
  if (existsSync(file))
656
473
  chmodSync(file, 0o600);
657
474
  }
658
475
  chmodSync(dirname(path), 0o700);
659
- // Full copies of the same secrets, one directory down.
660
476
  if (existsSync(backupsDir(path)))
661
477
  chmodSync(backupsDir(path), 0o700);
662
478
  }