@timqi/pier 0.0.29 → 0.1.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (165) hide show
  1. package/README.md +58 -125
  2. package/dist/agent/config.js +24 -11
  3. package/dist/agent/credentials.js +11 -23
  4. package/dist/agent/events.js +43 -62
  5. package/dist/agent/listing.js +113 -68
  6. package/dist/agent/pi.js +204 -211
  7. package/dist/boards/boards.js +19 -29
  8. package/dist/channels/attach.js +14 -42
  9. package/dist/channels/chains.js +33 -37
  10. package/dist/channels/chunk.js +8 -28
  11. package/dist/channels/commands.js +3 -14
  12. package/dist/channels/config.js +33 -52
  13. package/dist/channels/control.js +4 -13
  14. package/dist/channels/conversations.js +8 -25
  15. package/dist/channels/dedup.js +8 -17
  16. package/dist/channels/gatekeeper.js +13 -23
  17. package/dist/channels/lark-api.js +23 -63
  18. package/dist/channels/lark-outbound.js +12 -44
  19. package/dist/channels/lark-panel.js +8 -24
  20. package/dist/channels/lark-render.js +18 -62
  21. package/dist/channels/lark.js +52 -141
  22. package/dist/channels/lines.js +13 -15
  23. package/dist/channels/panel.js +16 -36
  24. package/dist/channels/receipts.js +29 -52
  25. package/dist/channels/routes.js +3 -9
  26. package/dist/channels/runtime.js +12 -23
  27. package/dist/channels/slack-api.js +34 -86
  28. package/dist/channels/slack-directory.js +7 -23
  29. package/dist/channels/slack-outbound.js +12 -56
  30. package/dist/channels/slack-panel.js +4 -13
  31. package/dist/channels/slack-render.js +23 -91
  32. package/dist/channels/slack-tool.js +48 -171
  33. package/dist/channels/slack.js +73 -239
  34. package/dist/channels/telegram-api.js +8 -20
  35. package/dist/channels/telegram-panel.js +5 -21
  36. package/dist/channels/telegram-render.js +13 -40
  37. package/dist/channels/telegram.js +54 -146
  38. package/dist/channels/types.js +5 -16
  39. package/dist/cli.js +17 -41
  40. package/dist/config-sync.js +87 -4
  41. package/dist/core/hub.js +7 -20
  42. package/dist/core/identity.js +20 -59
  43. package/dist/core/inbound-file.js +15 -49
  44. package/dist/core/inbox.js +13 -35
  45. package/dist/core/queue.js +3 -5
  46. package/dist/core/reply.js +41 -142
  47. package/dist/core/router.js +209 -260
  48. package/dist/core/types.js +17 -1
  49. package/dist/db.js +98 -252
  50. package/dist/drain.js +58 -51
  51. package/dist/extensions/index.js +3 -11
  52. package/dist/extensions/web/anthropic.js +3 -9
  53. package/dist/extensions/web/artifacts.js +2 -5
  54. package/dist/extensions/web/content.js +6 -14
  55. package/dist/extensions/web/http.js +2 -6
  56. package/dist/extensions/web/language.js +8 -18
  57. package/dist/extensions/web/openai.js +1 -1
  58. package/dist/extensions/web/provider.js +5 -18
  59. package/dist/extensions/web/tools.js +19 -63
  60. package/dist/lock.js +98 -0
  61. package/dist/log.js +9 -26
  62. package/dist/main.js +87 -179
  63. package/dist/paths.js +10 -26
  64. package/dist/secrets.js +19 -46
  65. package/dist/service.js +33 -75
  66. package/dist/settings.js +42 -65
  67. package/dist/tasks/agent.js +129 -114
  68. package/dist/tasks/callbacks.js +9 -19
  69. package/dist/tasks/command.js +29 -14
  70. package/dist/tasks/definitions.js +39 -62
  71. package/dist/tasks/execution.js +46 -42
  72. package/dist/tasks/groups.js +41 -35
  73. package/dist/tasks/messages.js +121 -182
  74. package/dist/tasks/outbox.js +61 -55
  75. package/dist/tasks/routes.js +5 -11
  76. package/dist/tasks/runs.js +14 -13
  77. package/dist/tasks/service.js +53 -54
  78. package/dist/tasks/store.js +53 -30
  79. package/dist/tasks/tool.js +132 -61
  80. package/dist/tools-task.js +20 -60
  81. package/dist/tools.js +100 -327
  82. package/dist/update.js +21 -44
  83. package/dist/web/auth.js +118 -179
  84. package/dist/web/config-sync.js +2 -2
  85. package/dist/web/config.js +3 -7
  86. package/dist/web/explorer.js +10 -21
  87. package/dist/web/fs.js +20 -42
  88. package/dist/web/instance.js +45 -85
  89. package/dist/web/providers.js +14 -13
  90. package/dist/web/public/assets/{activity-D3m4L2IL.js → activity-B89_hH7q.js} +2 -2
  91. package/dist/web/public/assets/activity-B89_hH7q.js.br +0 -0
  92. package/dist/web/public/assets/activity-B89_hH7q.js.gz +0 -0
  93. package/dist/web/public/assets/boards-BeKW0ZXK.js +1 -0
  94. package/dist/web/public/assets/boards-BeKW0ZXK.js.br +0 -0
  95. package/dist/web/public/assets/boards-BeKW0ZXK.js.gz +0 -0
  96. package/dist/web/public/assets/explorer-DIuMlaV3.js +4 -0
  97. package/dist/web/public/assets/explorer-DIuMlaV3.js.br +0 -0
  98. package/dist/web/public/assets/explorer-DIuMlaV3.js.gz +0 -0
  99. package/dist/web/public/assets/index-DzXDXra_.js +85 -0
  100. package/dist/web/public/assets/index-DzXDXra_.js.br +0 -0
  101. package/dist/web/public/assets/index-DzXDXra_.js.gz +0 -0
  102. package/dist/web/public/assets/index-eqQLVS8Q.css +2 -0
  103. package/dist/web/public/assets/index-eqQLVS8Q.css.br +0 -0
  104. package/dist/web/public/assets/index-eqQLVS8Q.css.gz +0 -0
  105. package/dist/web/public/assets/runs-Cwy0mN8i.js +1 -0
  106. package/dist/web/public/assets/runs-Cwy0mN8i.js.br +0 -0
  107. package/dist/web/public/assets/runs-Cwy0mN8i.js.gz +0 -0
  108. package/dist/web/public/assets/settings-DzZLmujq.js +5 -0
  109. package/dist/web/public/assets/settings-DzZLmujq.js.br +0 -0
  110. package/dist/web/public/assets/settings-DzZLmujq.js.gz +0 -0
  111. package/dist/web/public/assets/task-runs-BCakxFk8.js +3 -0
  112. package/dist/web/public/assets/task-runs-BCakxFk8.js.br +0 -0
  113. package/dist/web/public/assets/task-runs-BCakxFk8.js.gz +0 -0
  114. package/dist/web/public/assets/tasks-BlzEbk11.js +4 -0
  115. package/dist/web/public/assets/tasks-BlzEbk11.js.br +0 -0
  116. package/dist/web/public/assets/tasks-BlzEbk11.js.gz +0 -0
  117. package/dist/web/public/index.html +100 -130
  118. package/dist/web/public/index.html.br +0 -0
  119. package/dist/web/public/index.html.gz +0 -0
  120. package/dist/web/public/manifest.webmanifest +2 -2
  121. package/dist/web/public/manifest.webmanifest.br +0 -0
  122. package/dist/web/public/manifest.webmanifest.gz +0 -0
  123. package/dist/web/public/sw.js +14 -2
  124. package/dist/web/public/sw.js.br +0 -0
  125. package/dist/web/public/sw.js.gz +0 -0
  126. package/dist/web/push.js +55 -77
  127. package/dist/web/route.js +3 -7
  128. package/dist/web/server.js +131 -180
  129. package/dist/web/session-state.js +14 -54
  130. package/dist/web/types.js +2 -4
  131. package/dist/web/webpush.js +10 -25
  132. package/docs/deploy.md +115 -330
  133. package/package.json +2 -1
  134. package/skills/pier-boards/SKILL.md +81 -160
  135. package/skills/pier-help/SKILL.md +23 -20
  136. package/skills/pier-slack/SKILL.md +2 -2
  137. package/skills/pier-tasks/SKILL.md +153 -160
  138. package/dist/config-sync-fetch.js +0 -84
  139. package/dist/limits.js +0 -14
  140. package/dist/web/public/assets/activity-D3m4L2IL.js.br +0 -0
  141. package/dist/web/public/assets/activity-D3m4L2IL.js.gz +0 -0
  142. package/dist/web/public/assets/boards-BIObcQeX.js +0 -1
  143. package/dist/web/public/assets/boards-BIObcQeX.js.br +0 -0
  144. package/dist/web/public/assets/boards-BIObcQeX.js.gz +0 -0
  145. package/dist/web/public/assets/explorer-C_rSWPNB.js +0 -4
  146. package/dist/web/public/assets/explorer-C_rSWPNB.js.br +0 -0
  147. package/dist/web/public/assets/explorer-C_rSWPNB.js.gz +0 -0
  148. package/dist/web/public/assets/index-CX3fYZY5.css +0 -2
  149. package/dist/web/public/assets/index-CX3fYZY5.css.br +0 -0
  150. package/dist/web/public/assets/index-CX3fYZY5.css.gz +0 -0
  151. package/dist/web/public/assets/index-uFsZkKOQ.js +0 -85
  152. package/dist/web/public/assets/index-uFsZkKOQ.js.br +0 -0
  153. package/dist/web/public/assets/index-uFsZkKOQ.js.gz +0 -0
  154. package/dist/web/public/assets/runs-Ch6DZq6O.js +0 -1
  155. package/dist/web/public/assets/runs-Ch6DZq6O.js.br +0 -0
  156. package/dist/web/public/assets/runs-Ch6DZq6O.js.gz +0 -0
  157. package/dist/web/public/assets/settings-BWcEIEcv.js +0 -5
  158. package/dist/web/public/assets/settings-BWcEIEcv.js.br +0 -0
  159. package/dist/web/public/assets/settings-BWcEIEcv.js.gz +0 -0
  160. package/dist/web/public/assets/task-runs-DPkwv2UE.js +0 -3
  161. package/dist/web/public/assets/task-runs-DPkwv2UE.js.br +0 -0
  162. package/dist/web/public/assets/task-runs-DPkwv2UE.js.gz +0 -0
  163. package/dist/web/public/assets/tasks-DTiCi2mH.js +0 -4
  164. package/dist/web/public/assets/tasks-DTiCi2mH.js.br +0 -0
  165. package/dist/web/public/assets/tasks-DTiCi2mH.js.gz +0 -0
package/dist/db.js CHANGED
@@ -1,44 +1,25 @@
1
- // The one connection, and the one place the schema is written down.
2
- //
3
- // Every store used to open `pier.db` for itself and create its own tables with
4
- // `CREATE TABLE IF NOT EXISTS`. That works exactly once: it can add a table but
5
- // never change one, so the first column an upgrade needed would have left every
6
- // existing instance with a schema nothing could repair. `user_version` is a
7
- // single number per *database*, not per table, which is why the schema cannot
8
- // stay spread across five modules — and five connections to one file is also
9
- // five writers competing for the same lock.
10
- //
11
- // So: one connection, one ordered list of migrations, applied in one
12
- // transaction before any store exists. A store receives the handle and owns
13
- // only its queries.
1
+ // The one connection, and the one place the schema is written down: one ordered
2
+ // list of migrations, applied in one transaction before any store exists.
3
+ // `user_version` is per database, not per table, so the schema cannot be
4
+ // spread across modules.
14
5
  import { chmodSync, existsSync, mkdirSync, readdirSync, renameSync, rmSync, statSync } from "node:fs";
15
6
  import { basename, dirname, join } from "node:path";
16
7
  import { DatabaseSync } from "node:sqlite";
17
8
  import { logger } from "./log.js";
18
9
  import { PIER_DB } from "./paths.js";
19
10
  const log = logger("db");
20
- /** Snapshots to keep *of each kind*. Three is two upgrades of regret plus one:
21
- * they are full copies of the database, and the one that matters is the
22
- * newest. Counted per kind because the two kinds answer different questions —
23
- * a run of releases must not evict the pre-migration copies. */
11
+ /** Per kind, because full copies are large and a run of releases must not
12
+ * evict the pre-migration copies. */
24
13
  const KEEP_BACKUPS = 3;
25
- /** How long a second process may wait for the write lock before failing. Two
26
- * Pier processes on one PIER_HOME contend exactly once — at boot, when both
27
- * want to migrate — and failing instantly there turns a restart race into a
28
- * crash loop. */
14
+ /** Two Pier processes on one PIER_HOME contend at boot, when both want to
15
+ * migrate; failing instantly turns a restart race into a crash loop. */
29
16
  const BUSY_TIMEOUT_MS = 5_000;
30
- /**
31
- * Append-only, never edited: index + 1 is the `user_version` a database is at
32
- * once that entry has run. An entry that shipped is history — fix a mistake
33
- * with the next one, because somebody's database already ran the old one.
34
- *
35
- * Migration 1 is the whole schema as of 0.0.1 and assumes nothing before it:
36
- * pre-release databases are not upgraded, they are deleted.
37
- */
17
+ /** Append-only, never edited: index + 1 is the `user_version` after that entry.
18
+ * Fix a mistake with the next one — somebody's database already ran the old one. */
38
19
  const MIGRATIONS = [
39
20
  // 1 — the 0.0.1 schema.
40
21
  `
41
- -- The single credential in front of every HTTP surface (web/auth.ts).
22
+ -- The single credential in front of every HTTP surface.
42
23
  CREATE TABLE auth (
43
24
  id INTEGER PRIMARY KEY CHECK (id = 1),
44
25
  salt TEXT NOT NULL,
@@ -46,15 +27,13 @@ const MIGRATIONS = [
46
27
  created_at INTEGER NOT NULL
47
28
  );
48
29
 
49
- -- Instance facts that are neither a credential nor per-session; one row per
50
- -- setting, so the next setting is not the next table.
30
+ -- Instance facts that are neither a credential nor per-session.
51
31
  CREATE TABLE settings (
52
32
  key TEXT PRIMARY KEY,
53
33
  value TEXT NOT NULL
54
34
  );
55
35
 
56
- -- Workbench bookkeeping: pinned = listed under Projects, unread = a turn
57
- -- finished that no client has acknowledged.
36
+ -- Workbench bookkeeping; unread = a turn finished that no client acknowledged.
58
37
  CREATE TABLE session_state (
59
38
  session_id TEXT PRIMARY KEY,
60
39
  pinned INTEGER NOT NULL DEFAULT 0,
@@ -87,8 +66,8 @@ const MIGRATIONS = [
87
66
  PRIMARY KEY (platform, chat_id, message_id)
88
67
  );
89
68
 
90
- -- Scheduled work. The row keeps its whole JSON document; the columns beside
91
- -- it are only what a query filters or orders by.
69
+ -- Scheduled work: the JSON is the document, the columns beside it are only
70
+ -- what a query filters or orders by.
92
71
  CREATE TABLE tasks (
93
72
  id TEXT PRIMARY KEY,
94
73
  updated_at INTEGER NOT NULL,
@@ -119,10 +98,9 @@ const MIGRATIONS = [
119
98
  json TEXT NOT NULL
120
99
  );
121
100
  `,
122
- // 2 — provider credentials move from <agentDir>/auth.json into the database.
101
+ // 2 — provider credentials.
123
102
  `
124
103
  -- One row per provider (key = provider id), value sealed by secrets.ts.
125
- -- Owned by agent/credentials.ts.
126
104
  CREATE TABLE credentials (
127
105
  key TEXT PRIMARY KEY,
128
106
  value TEXT NOT NULL
@@ -130,8 +108,7 @@ const MIGRATIONS = [
130
108
  `,
131
109
  // 3 — what a restart's drain deadline cut off, told to the chat at next boot.
132
110
  `
133
- -- Written only when a graceful restart aborts a still-running turn; the next
134
- -- boot delivers each row and clears it once delivered. Owned by drain.ts.
111
+ -- Turns a graceful restart cut off; each row is delivered at next boot, then cleared.
135
112
  CREATE TABLE restart_ledger (
136
113
  id INTEGER PRIMARY KEY,
137
114
  channel_id TEXT NOT NULL,
@@ -146,10 +123,9 @@ const MIGRATIONS = [
146
123
  ALTER TABLE session_state ADD COLUMN title TEXT;
147
124
  ALTER TABLE session_state ADD COLUMN created_at INTEGER;
148
125
  `,
149
- // 5 — the workbench can reach a browser that is not open (web/push.ts).
126
+ // 5 — Web Push.
150
127
  `
151
- -- One row per browser that asked to be notified, exactly as the Push API
152
- -- described it; a dead endpoint is deleted when its service says so.
128
+ -- One row per browser that asked to be notified, as the Push API described it.
153
129
  CREATE TABLE push_subscriptions (
154
130
  endpoint TEXT PRIMARY KEY,
155
131
  p256dh TEXT NOT NULL,
@@ -158,8 +134,8 @@ const MIGRATIONS = [
158
134
  created_at INTEGER NOT NULL
159
135
  );
160
136
 
161
- -- This instance's VAPID identity: one key pair, minted on first use. Every
162
- -- subscription above is bound to it, so it is never rotated on its own.
137
+ -- This instance's VAPID key pair; every subscription is bound to it, so it
138
+ -- is never rotated on its own.
163
139
  CREATE TABLE push_identity (
164
140
  id INTEGER PRIMARY KEY CHECK (id = 1),
165
141
  public_key TEXT NOT NULL,
@@ -167,21 +143,16 @@ const MIGRATIONS = [
167
143
  created_at INTEGER NOT NULL
168
144
  );
169
145
  `,
170
- // 6 — Projects keeps the order the workbench was put in, by hand.
146
+ // 6 — manual order in the rail.
171
147
  `
172
- -- Manual order, both nullable: a row nobody has dragged sorts on top of the
173
- -- list it belongs to, so a fresh database needs no backfill. sort places a
174
- -- session inside its project; project_sort places the project, carried on
175
- -- every one of its rows because a project is a cwd, not a table.
148
+ -- NULL sorts on top, so a fresh database needs no backfill.
176
149
  ALTER TABLE session_state ADD COLUMN sort INTEGER;
177
150
  ALTER TABLE session_state ADD COLUMN project_sort INTEGER;
178
151
  `,
179
- // 7 — the session listing, so a transcript is read once (agent/listing.ts).
152
+ // 7 — the session listing, so a transcript is read once.
180
153
  `
181
- -- One row per session file. (size, mtime) is what makes the row usable
182
- -- without opening the file; parsed_bytes is where reading resumes when it
183
- -- grew, and is always a line boundary. Derived from disk and disposable: a
184
- -- deleted row costs one re-read, never a fact.
154
+ -- One row per session file, derived from disk and disposable. A row whose
155
+ -- (size, mtime) match is trusted unopened; parsed_bytes is where reading resumes, always a line boundary.
185
156
  CREATE TABLE session_index (
186
157
  path TEXT PRIMARY KEY,
187
158
  id TEXT NOT NULL,
@@ -194,73 +165,39 @@ const MIGRATIONS = [
194
165
  parsed_bytes INTEGER NOT NULL
195
166
  );
196
167
  `,
197
- // 8 — Projects holds a working set: what is warm, plus what is kept.
168
+ // 8 — working-set lease columns (dropped again in 9 and 11).
198
169
  `
199
- -- Membership was permanent, so every throwaway session stayed in the rail
200
- -- until someone removed it by hand. last_active is the lease: the end of a
201
- -- turn renews it, and web/session-state.ts stops listing a row that ran out.
202
- -- kept opts one row out of expiry entirely — what the pin control now means.
203
170
  ALTER TABLE session_state ADD COLUMN kept INTEGER NOT NULL DEFAULT 0;
204
171
  ALTER TABLE session_state ADD COLUMN last_active INTEGER;
205
- -- Left NULL on purpose: the honest value is when the transcript was last
206
- -- written, which only a listing knows. web/server.ts pays one for a database
207
- -- carrying rows without it, the same gate the pin backfill already uses, so
208
- -- the first rail after an upgrade is dated by use and not by creation.
209
172
  `,
210
- // 9 — the summary a transcript already carries is read, not mirrored.
173
+ // 9 — drop the columns that mirrored the transcript.
211
174
  `
212
- -- Dropped rather than left unread: a column nobody writes still answers when
213
- -- somebody selects it, and the next reader has no way to tell a stale title
214
- -- from a current one. The pre-migration backup beside the database is the
215
- -- way back, not a row of fossils. cwd stays — it is the key a project's
216
- -- manual place is stamped on, and it never changes for a session.
217
175
  ALTER TABLE session_state DROP COLUMN title;
218
176
  ALTER TABLE session_state DROP COLUMN created_at;
219
177
  ALTER TABLE session_state DROP COLUMN last_active;
220
178
  `,
221
- // 10 — taking a session into Projects is itself an act, and it is dated.
179
+ // 10 — pinned_at (dropped again in 11).
222
180
  `
223
- -- When a hand last put this session in Projects (pin, or a keep toggle).
224
- -- Not the mirror migration 9 removed: last_active was a copy of a fact the
225
- -- transcript owns, while this one exists nowhere else — pinning a cold
226
- -- session back is a statement that it is warm again, and without a record of
227
- -- *when* it was made the row is dropped by the same read that drew it.
228
- -- NULL for every row that predates this: never pinned within a lease.
229
181
  ALTER TABLE session_state ADD COLUMN pinned_at INTEGER;
230
182
  `,
231
- // 11 — Projects holds what a hand put there, for as long as the hand says.
183
+ // 11 — the lease is gone, so both of its columns are.
232
184
  `
233
- -- The lease is gone, so both of its columns are. It expired nothing: a row
234
- -- it dropped kept its transcript, its place and its ownership, and one more
235
- -- turn brought it back — so what it actually did was hide rows nobody asked
236
- -- it to hide, and kept existed only to opt out of that. On the instance this
237
- -- was decided on, the lease had never dropped a row: 20 pinned sessions,
238
- -- none past seven days, one kept. Removing a row from Projects is the ✓ on
239
- -- the row, and it stays the only way out.
240
185
  ALTER TABLE session_state DROP COLUMN kept;
241
186
  ALTER TABLE session_state DROP COLUMN pinned_at;
242
187
  `,
243
- // 12 — one row is how two processes take turns (src/tools.ts).
188
+ // 12 — the cross-process tools sync lock.
244
189
  `
245
- -- The tools sync, held across processes: the token says who holds it, the
246
- -- heartbeat says they are still alive. Both processes already open this
247
- -- database, and BEGIN IMMEDIATE is real mutual exclusion — a lock file with
248
- -- a pid in it is neither, which is what this replaces. One row, because
249
- -- there is one thing to serialize; the second lock can bring its own table
250
- -- and its own reason for existing.
190
+ -- One row: token says who holds the lock, heartbeat_at says they are still alive.
251
191
  CREATE TABLE tools_sync_lock (
252
192
  id INTEGER PRIMARY KEY CHECK (id = 1),
253
193
  token TEXT NOT NULL,
254
194
  heartbeat_at INTEGER NOT NULL
255
195
  );
256
196
  `,
257
- // 13 — a signed-in browser can be signed out on its own (web/auth.ts).
197
+ // 13 — web sessions.
258
198
  `
259
- -- One row per signed-in browser. The cookie carries "<id>.<token>" and only
260
- -- the token's SHA-256 is stored, so a copy of this database cannot be turned
261
- -- into a session — and deleting a row is what revocation is. seen_at is the
262
- -- whole lifetime: the session ends one TTL after it, so there is no second
263
- -- column that can disagree about when.
199
+ -- One row per signed-in browser; only the cookie token's SHA-256 is stored,
200
+ -- deleting a row is revocation, and the session ends one TTL after seen_at.
264
201
  CREATE TABLE web_sessions (
265
202
  id TEXT PRIMARY KEY,
266
203
  token_hash TEXT NOT NULL,
@@ -270,15 +207,9 @@ const MIGRATIONS = [
270
207
  agent TEXT NOT NULL
271
208
  );
272
209
  `,
273
- // 14 — signing a browser out also stops notifying it (web/push.ts).
210
+ // 14 — a push subscription belongs to the web session that made it.
274
211
  `
275
- -- A subscription belongs to the web session that made it, and dies with it:
276
- -- the cascade is the rule, so no code has to remember to run it — revoking a
277
- -- session, changing the password and recovering it all reach here for free.
278
- -- Rebuilt rather than altered because a foreign key cannot be added to an
279
- -- existing table; nothing is carried over, since migration 13 invalidated
280
- -- every cookie and each of these rows belongs to a browser that is now
281
- -- signed out. A browser re-subscribes on its next load.
212
+ -- Rebuilt: a foreign key cannot be added to an existing table.
282
213
  DROP TABLE push_subscriptions;
283
214
  CREATE TABLE push_subscriptions (
284
215
  endpoint TEXT PRIMARY KEY,
@@ -289,23 +220,14 @@ const MIGRATIONS = [
289
220
  session_id TEXT NOT NULL REFERENCES web_sessions(id) ON DELETE CASCADE
290
221
  );
291
222
  `,
292
- // 15 — the scheduler's tick stops reading the rows it cannot deliver.
223
+ // 15 — indexes for the scheduler's once-a-second sweep.
293
224
  `
294
- -- Once a second, tasks/ asks for the runs whose callback is still owed and
295
- -- the messages whose injection has not landed. Both are a handful of rows
296
- -- filtered on one low-cardinality column, and without an index both are a
297
- -- full scan of a table that only grows — the sweep got slower with every
298
- -- run that finished cleanly and can never match again.
299
225
  CREATE INDEX task_runs_callback_state ON task_runs(callback_state);
300
226
  CREATE INDEX task_messages_state ON task_messages(state);
301
227
  `,
302
- // 16 — the same tick stops parsing every task document to find none due.
228
+ // 16 — an indexable next-due value, with the JSON still the only record.
303
229
  `
304
- -- When a task is next due is the one field the scheduler asks about once a
305
- -- second, and it lived only inside the JSON: answering meant reading and
306
- -- parsing every definition, due or not. A generated column keeps the JSON
307
- -- as the only record while giving the query planner a value it can index; a
308
- -- disabled or archived task has no next run, so NULLs stay out of the index.
230
+ -- A disabled or archived task has no next run, so NULLs stay out of the index.
309
231
  ALTER TABLE tasks ADD COLUMN next_run_at INTEGER
310
232
  GENERATED ALWAYS AS (json_extract(json, '$.nextRunAt')) VIRTUAL;
311
233
  CREATE INDEX tasks_due ON tasks(next_run_at) WHERE next_run_at IS NOT NULL;
@@ -316,49 +238,43 @@ const MIGRATIONS = [
316
238
  CREATE INDEX task_runs_visible_time ON task_runs(queued_at DESC, id DESC)
317
239
  WHERE NOT (state = 'succeeded' AND json_extract(json, '$.matched') IS 0);
318
240
  `,
319
- // 18 — opening a session lists every run it delegated, not the last hour's.
241
+ // 18 — runs by delegating session.
320
242
  `
321
- -- The run cards are the messages a session sent, so the transcript wants
322
- -- all of them: the query is by the delegating session, which lived only in
323
- -- the JSON — without this every session open was a full scan of task_runs.
324
243
  CREATE INDEX task_runs_invoked_by ON task_runs(json_extract(json, '$.invokedBySessionId'), queued_at DESC);
325
244
  `,
326
- // 19 — the rail is one flat list, and pinned means "stuck to the top".
245
+ // 19 — pinned now means "stuck to the top"; nobody put a historical session there.
327
246
  `
328
- -- pinned used to mean "listed under Projects", and every session created in
329
- -- the workbench was. Now it means on top of the list, and nobody put a
330
- -- historical session there: kept as-is, every web session ever made would
331
- -- land in the pinned section. project_sort stays as a column nothing reads
332
- -- — a SQLite column drop rewrites the table for a NULL nobody pays for.
333
247
  UPDATE session_state SET pinned = 0;
334
248
  `,
335
- // 20 — the top of the rail is maintained, not arranged: no pin, no drag.
249
+ // 20 — sort becomes the rank in the working set the rail keeps on top; pinned goes.
336
250
  `
337
- -- sort was the place a hand dragged a pinned row to; it is now the rank in
338
- -- the working set the rail keeps on top, which a session enters by being
339
- -- spoken to and leaves by being pushed out of the last slot. The pinned rows
340
- -- are that set's first members — they are what somebody was working on — in
341
- -- the order they were arranged in; never-dragged ones sorted first, so -1 is
342
- -- the rank that keeps them there.
251
+ -- Pinned rows seed the set at rank -1, capped at the set's size (8).
343
252
  UPDATE session_state SET sort = -1 WHERE pinned = 1 AND sort IS NULL;
344
253
  UPDATE session_state SET sort = NULL WHERE pinned = 0;
345
- -- The set has a size (web/session-state.ts); more pins than that is a list
346
- -- the promotion rule would never have built.
347
254
  UPDATE session_state SET sort = NULL WHERE session_id IN (
348
255
  SELECT session_id FROM session_state WHERE sort IS NOT NULL
349
256
  ORDER BY sort, session_id LIMIT -1 OFFSET 8
350
257
  );
351
258
  ALTER TABLE session_state DROP COLUMN pinned;
259
+ `,
260
+ // 21 — unread is only written for web sessions now; clear the marks nobody could ack.
261
+ `
262
+ UPDATE session_state SET unread = 0;
263
+ `,
264
+ // 22 — message search.
265
+ `
266
+ -- One row per user message and assistant reply, keyed by transcript path.
267
+ -- Trigram: substring match with no word segmentation, which CJK text has none of.
268
+ CREATE VIRTUAL TABLE session_fts USING fts5(
269
+ text, session_id UNINDEXED, path UNINDEXED, role UNINDEXED, at UNINDEXED,
270
+ tokenize = 'trigram'
271
+ );
272
+ -- Every row goes so the next scan re-reads each transcript and fills the table above.
273
+ DELETE FROM session_index;
352
274
  `,
353
275
  ];
354
- /**
355
- * Several writes as one, or none. `BEGIN IMMEDIATE` because every writer here
356
- * competes with another Pier process on the same file: taking the write lock
357
- * up front turns a race into a wait, where deferred would turn it into
358
- * SQLITE_BUSY halfway through. The rollback is the reason this is shared —
359
- * three modules had written the same seven lines, and a `catch` that forgets
360
- * to roll back leaves the connection in a transaction forever.
361
- */
276
+ /** `BEGIN IMMEDIATE`: taking the write lock up front turns a race with another
277
+ * Pier process into a wait, where deferred would be SQLITE_BUSY halfway through. */
362
278
  export function transact(db, work) {
363
279
  db.exec("BEGIN IMMEDIATE");
364
280
  try {
@@ -371,15 +287,8 @@ export function transact(db, work) {
371
287
  throw err;
372
288
  }
373
289
  }
374
- /**
375
- * Prepared statements, memoized by their SQL. `prepare()` compiles, and the
376
- * callers here hand it the same handful of strings forever — twice per
377
- * authenticated request, once a second per scheduler sweep. A `StatementSync`
378
- * is reusable with different bound parameters, so one per SQL string per
379
- * connection is the whole cache. Bound to the connection because a statement
380
- * belongs to the database that compiled it; SQL built per call does not
381
- * belong in here, since the cache would then grow without a bound.
382
- */
290
+ /** Prepared statements memoized by SQL; callers hand it the same handful of
291
+ * strings forever. SQL built per call does not belong here: the cache is unbounded. */
383
292
  export function statements(db) {
384
293
  const cache = new Map();
385
294
  return (sql) => {
@@ -390,28 +299,15 @@ export function statements(db) {
390
299
  };
391
300
  }
392
301
  let shared;
393
- /**
394
- * The process's one connection, opened and migrated on first use. Every store
395
- * defaults to it; a test passes `openDb(":memory:")` instead.
396
- */
302
+ /** The process's one connection; a test passes `openDb(":memory:")` instead. */
397
303
  export const pierDb = () => (shared ??= openDb(PIER_DB));
398
- /** A release-level restore point, taken for every release even when it has no
399
- * schema migration. The previous complete copies stay put if writing this one
400
- * fails, and the service may be running while it is written: `copyDatabase`
401
- * reads through a read-only connection, so what it writes is one consistent
402
- * snapshot of a live database rather than a torn `cp`.
403
- *
404
- * `version` is the Pier that produced this database, not the one being
405
- * installed: the updater runs this from the tree it is about to replace, and
406
- * restoring a database means reinstalling the code that speaks its schema
407
- * (`migrate` refuses one from a newer Pier). So the name carries the other half
408
- * of the pair. Backing up twice at one version replaces that version's copy —
409
- * the pairing is identical, so a second name for it would say nothing. */
304
+ /** A restore point per release, schema migration or not. `version` is the Pier
305
+ * that produced the database: restoring one means reinstalling the code that
306
+ * speaks its schema, so the name carries that half of the pair. */
410
307
  export function backupDb(version, path = PIER_DB) {
411
308
  if (!existsSync(path))
412
309
  return undefined;
413
- // In a filename, so it may not carry a separator or a traversal; a version
414
- // this malformed is a broken install, not something to guess at.
310
+ // In a filename, so no separator or traversal.
415
311
  const safe = version.replaceAll(/[^0-9A-Za-z.+-]/g, "_") || "unknown";
416
312
  const bak = join(backupsDir(path, true), `${basename(path)}.release-${safe}.bak`);
417
313
  copyDatabase(path, bak);
@@ -419,67 +315,49 @@ export function backupDb(version, path = PIER_DB) {
419
315
  prune(releases(path));
420
316
  return bak;
421
317
  }
422
- /** Open a database, bring it to the current schema, and lock down its files.
423
- * `migrations` is injectable only so tests can exercise an upgrade — there is
424
- * exactly one real list. */
318
+ /** `migrations` is injectable only so tests can exercise an upgrade. */
425
319
  export function openDb(path, migrations = MIGRATIONS) {
426
320
  if (path !== ":memory:")
427
321
  mkdirSync(dirname(path), { recursive: true, mode: 0o700 });
428
322
  const db = new DatabaseSync(path);
429
323
  // Timeout first: two processes booting together contend on the WAL switch
430
- // itself. Outside the transaction below: journal_mode is a property of the
431
- // file, and SQLite refuses to change it inside one.
324
+ // itself, and journal_mode cannot change inside a transaction.
432
325
  db.exec(`PRAGMA busy_timeout = ${BUSY_TIMEOUT_MS}`);
433
326
  db.exec("PRAGMA journal_mode = WAL");
434
- // WAL's default leaves every commit waiting on an fsync, and DatabaseSync is
435
- // synchronous — that wait is the event loop's. NORMAL still survives a
436
- // process crash; only a power loss can cost the last transactions, which for
437
- // routing state and task bookkeeping is a fair trade for not blocking.
327
+ // DatabaseSync is synchronous, so WAL's per-commit fsync would be the event
328
+ // loop's wait. NORMAL survives a process crash; only power loss costs anything.
438
329
  db.exec("PRAGMA synchronous = NORMAL");
439
- // Off by default in SQLite, and a declared relationship nothing enforces is
440
- // a comment. Set before migrate(): it is a per-connection switch and a no-op
441
- // inside a transaction. Nothing older declares a key, so this changes the
442
- // behaviour of exactly one table — push_subscriptions, whose rows must not
443
- // outlive the session that made them.
330
+ // Off by default in SQLite; per connection, and a no-op inside a transaction.
444
331
  db.exec("PRAGMA foreign_keys = ON");
445
332
  migrate(db, path, migrations);
446
333
  if (path !== ":memory:")
447
334
  restrict(path);
448
335
  return db;
449
336
  }
450
- /**
451
- * Upgrades only. `user_version` counts up and nothing counts it back down, so a
452
- * database from a newer Pier is refused rather than served: the old code would
453
- * happily write the new schema's tables and lose whatever it did not know
454
- * about. The way back is the `.bak` this function writes before upgrading.
455
- */
337
+ /** Upgrades only: a database from a newer Pier is refused, since old code would
338
+ * write the new schema's tables and lose what it did not know about. */
456
339
  function migrate(db, path, migrations) {
457
340
  const { user_version: at } = db.prepare("PRAGMA user_version").get();
458
341
  const target = migrations.length;
459
342
  if (at > target) {
460
- // Name the snapshot that exists rather than a pattern: the operator is
461
- // reading this because the service will not start.
343
+ // Name the snapshot that exists: the operator is reading this because the
344
+ // service will not start.
462
345
  const newest = path === ":memory:" ? undefined : snapshots(path)[0]?.file;
463
346
  throw new Error(`${path} is at schema ${at}, this Pier speaks ${target}: a database is ` +
464
347
  `never downgraded. Restore ${newest ?? `a copy from ${backupsDir(path)}`}, or run the newer Pier.`);
465
348
  }
466
- // Version 0 with tables is a database from before versioning existed.
467
- // Migration 1 assumes an empty file, so the collision it would hit says
468
- // "table already exists" — this says what is actually wrong and what to do.
349
+ // Version 0 with tables predates versioning; migration 1 assumes an empty file.
469
350
  if (at === 0 && db.prepare("SELECT 1 FROM sqlite_master WHERE type = 'table' LIMIT 1").get()) {
470
351
  throw new Error(`${path} predates schema versioning and cannot be upgraded — nothing was changed. ` +
471
352
  `Move the file aside and restart; channel credentials, tasks and the password start over.`);
472
353
  }
473
354
  if (at === target)
474
355
  return;
475
- // One transaction for the statements *and* the version number: a crash
476
- // between them would leave a database whose version describes a schema it
477
- // does not have, which is worse than a crash.
356
+ // Statements and version number in one transaction: a version describing a
357
+ // schema the database does not have is worse than a crash.
478
358
  db.exec("BEGIN IMMEDIATE");
479
- // Re-read inside the lock. Two processes starting together both saw work to
480
- // do; the one that waited for the lock would otherwise replay migrations the
481
- // winner already committed and die on "table already exists", with a healthy
482
- // database in front of it.
359
+ // Re-read inside the lock: the process that waited must not replay what the
360
+ // winner already committed.
483
361
  const { user_version: locked } = db.prepare("PRAGMA user_version").get();
484
362
  if (locked >= target) {
485
363
  db.exec("ROLLBACK");
@@ -489,9 +367,8 @@ function migrate(db, path, migrations) {
489
367
  log.info(`schema already at ${locked}, migrated by another process`);
490
368
  return;
491
369
  }
492
- // Keep the write lock while a second, read-only connection copies the last
493
- // committed state. That connection may VACUUM while this one holds a RESERVED
494
- // lock; other Pier starts wait here instead of racing on the shared .tmp.
370
+ // Under the write lock: a read-only connection may VACUUM while this one
371
+ // holds RESERVED, and other Pier starts wait instead of racing on the .tmp.
495
372
  if (locked > 0 && path !== ":memory:") {
496
373
  try {
497
374
  snapshot(path, locked);
@@ -510,32 +387,22 @@ function migrate(db, path, migrations) {
510
387
  }
511
388
  catch (err) {
512
389
  db.exec("ROLLBACK");
513
- // Which one, and that the database is untouched: the operator's next move
514
- // is to restore a backup or to report the migration, and a bare SQLite
515
- // error says neither.
390
+ // Which one, and that the database is untouched: a bare SQLite error says neither.
516
391
  throw new Error(`migration ${step + 1} failed on ${path} — nothing was changed: ${String(err)}`, { cause: err });
517
392
  }
518
393
  log.info(locked === 0 ? `schema created at version ${target}` : `schema ${locked} → ${target}`);
519
394
  if (path !== ":memory:")
520
395
  prune(snapshots(path).map(({ file }) => file));
521
396
  }
522
- /**
523
- * The copy that exists because `user_version` only counts up: the transaction
524
- * above protects against a migration that *failed*, and this against one that
525
- * succeeded and should not have. `VACUUM INTO`, not `cp`: under WAL the
526
- * committed tail of the database lives in the `-wal` sidecar.
527
- *
528
- * Written under a temporary name and renamed into place. `VACUUM INTO` refuses
529
- * an existing target, so the alternative is deleting the previous snapshot
530
- * first — which means the likely failure here, a full disk, leaves neither the
531
- * old snapshot nor a complete new one. A rename is atomic: the `.bak` name only
532
- * ever refers to a finished copy.
533
- */
397
+ /** The transaction protects against a migration that failed; this against one
398
+ * that succeeded and should not have. */
534
399
  function snapshot(path, at) {
535
400
  const bak = join(backupsDir(path, true), `${basename(path)}.v${at}.bak`);
536
401
  copyDatabase(path, bak);
537
402
  log.info(`pre-migration backup: ${bak}`);
538
403
  }
404
+ /** `VACUUM INTO`, not `cp`: under WAL the committed tail lives in the `-wal`
405
+ * sidecar. Temp name plus rename, so a full disk leaves the old copy intact. */
539
406
  function copyDatabase(path, bak) {
540
407
  const tmp = `${bak}.tmp`;
541
408
  rmSync(tmp, { force: true }); // a previous crash may have left one
@@ -549,16 +416,9 @@ function copyDatabase(path, bak) {
549
416
  chmodSync(tmp, 0o600); // it holds everything the 0600 database holds
550
417
  renameSync(tmp, bak);
551
418
  }
552
- /**
553
- * One directory for every copy of this database, `db/backups/`. Beside the
554
- * database was fine while there was one snapshot per schema; a restore point
555
- * per release turns that into a listing where the live file and its sidecars
556
- * are hard to pick out, and "which of these do I not delete" is the wrong
557
- * question to make an operator answer under pressure.
558
- *
559
- * `create` also adopts what an older Pier wrote next to the database, so the
560
- * restore procedure names one location instead of two forever.
561
- */
419
+ /** `db/backups/`, so the live file and its sidecars are never in the listing an
420
+ * operator prunes under pressure. `create` adopts copies an older Pier left
421
+ * beside the database. */
562
422
  function backupsDir(path, create = false) {
563
423
  const dir = join(dirname(path), "backups");
564
424
  if (!create)
@@ -573,8 +433,7 @@ function backupsDir(path, create = false) {
573
433
  }
574
434
  return dir;
575
435
  }
576
- /** The copies of one kind: `v<schema>` or `release-<version>`. Disjoint
577
- * prefixes, so each kind is counted and pruned on its own. */
436
+ /** `v<schema>` or `release-<version>`: disjoint prefixes, pruned per kind. */
578
437
  function listBackups(path, kind) {
579
438
  const dir = backupsDir(path);
580
439
  if (!existsSync(dir))
@@ -582,8 +441,7 @@ function listBackups(path, kind) {
582
441
  const prefix = `${basename(path)}.${kind}`;
583
442
  return readdirSync(dir).filter((name) => name.startsWith(prefix) && name.endsWith(".bak"));
584
443
  }
585
- /** Pre-migration snapshots, newest schema first — the number in the name is an
586
- * ordinal, so it orders them without asking the filesystem. */
444
+ /** Pre-migration snapshots, newest schema first. */
587
445
  function snapshots(path) {
588
446
  const prefix = `${basename(path)}.v`;
589
447
  return listBackups(path, "v")
@@ -594,39 +452,27 @@ function snapshots(path) {
594
452
  .filter(({ version }) => Number.isInteger(version))
595
453
  .sort((a, b) => b.version - a.version);
596
454
  }
597
- /** Release restore points, newest copy first. Ordered by mtime: the name holds
598
- * a Pier version, and comparing those means reimplementing semver here — while
599
- * two updates of one instance are never in flight at the same moment. Legacy
600
- * `pier.db.release.bak` shares the prefix, so it ages out like the rest. */
455
+ /** Release restore points, newest first by mtime: ordering by the version in
456
+ * the name would mean reimplementing semver here. */
601
457
  function releases(path) {
602
458
  return listBackups(path, "release")
603
459
  .map((name) => join(backupsDir(path), name))
604
460
  .sort((a, b) => statSync(b).mtimeMs - statSync(a).mtimeMs);
605
461
  }
606
- /** Keep the newest few, oldest first out. Nobody restores a database from four
607
- * upgrades ago, and every one of these is the size of the whole database. */
608
462
  function prune(newestFirst) {
609
463
  for (const file of newestFirst.slice(KEEP_BACKUPS)) {
610
464
  rmSync(file, { force: true });
611
465
  log.info(`removed superseded backup: ${file}`);
612
466
  }
613
467
  }
614
- /**
615
- * The database holds the password hash, so it is not world-readable — and
616
- * neither are the sidecars, where a 0644 `-wal` would leak exactly what the
617
- * 0600 database is hiding. Done after the migration, so the sidecars that
618
- * writing created exist by now; SQLite gives later ones the database's mode.
619
- * The directory too: it exists only to hold this database and its sidecars
620
- * (paths.ts puts them under their own `db/`, away from the boards PIER_HOME
621
- * also holds), so nothing else needs to see into it.
622
- */
468
+ /** The database holds the password hash; a 0644 `-wal` would leak what the
469
+ * 0600 database hides. After the migration, so the sidecars exist by now. */
623
470
  function restrict(path) {
624
471
  for (const file of [path, `${path}-wal`, `${path}-shm`]) {
625
472
  if (existsSync(file))
626
473
  chmodSync(file, 0o600);
627
474
  }
628
475
  chmodSync(dirname(path), 0o700);
629
- // Full copies of the same secrets, one directory down.
630
476
  if (existsSync(backupsDir(path)))
631
477
  chmodSync(backupsDir(path), 0o700);
632
478
  }