@harperfast/harper 5.2.0-alpha.4 → 5.2.0-alpha.6

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (173) hide show
  1. package/agent/agent.ts +152 -10
  2. package/agent/bestPractices.ts +58 -0
  3. package/agent/mcpTools.ts +122 -0
  4. package/agent/operations.ts +10 -1
  5. package/agent/registryTools.ts +117 -0
  6. package/agent/session.ts +16 -7
  7. package/agent/tools/fsTools.ts +74 -27
  8. package/agent/tools/inspectorTool.ts +459 -0
  9. package/agent/toolset.ts +27 -7
  10. package/agent/types.ts +2 -0
  11. package/components/Application.ts +566 -85
  12. package/components/OptionsWatcher.ts +62 -2
  13. package/components/Scope.ts +31 -8
  14. package/components/componentLoader.ts +5 -1
  15. package/components/componentSecrets.ts +514 -55
  16. package/components/deploymentRecorder.ts +7 -6
  17. package/components/gitCredentialHelper.js +115 -0
  18. package/components/gitCredentialServer.ts +251 -0
  19. package/components/mcp/tools/application.ts +197 -23
  20. package/components/mcp/tools/operations.ts +1 -1
  21. package/components/mcp/tools/schemas/derive.ts +6 -2
  22. package/components/operations.js +21 -19
  23. package/components/operationsValidation.js +86 -21
  24. package/components/secretOperations.ts +110 -30
  25. package/config/harperConfigEnvVars.ts +90 -5
  26. package/config-root.schema.json +4 -0
  27. package/dist/agent/agent.d.ts +24 -3
  28. package/dist/agent/agent.js +172 -10
  29. package/dist/agent/agent.js.map +1 -1
  30. package/dist/agent/bestPractices.d.ts +24 -0
  31. package/dist/agent/bestPractices.js +60 -0
  32. package/dist/agent/bestPractices.js.map +1 -0
  33. package/dist/agent/mcpTools.d.ts +24 -0
  34. package/dist/agent/mcpTools.js +115 -0
  35. package/dist/agent/mcpTools.js.map +1 -0
  36. package/dist/agent/operations.js +10 -1
  37. package/dist/agent/operations.js.map +1 -1
  38. package/dist/agent/registryTools.d.ts +45 -0
  39. package/dist/agent/registryTools.js +113 -0
  40. package/dist/agent/registryTools.js.map +1 -0
  41. package/dist/agent/session.js +16 -7
  42. package/dist/agent/session.js.map +1 -1
  43. package/dist/agent/tools/fsTools.js +70 -28
  44. package/dist/agent/tools/fsTools.js.map +1 -1
  45. package/dist/agent/tools/inspectorTool.d.ts +40 -0
  46. package/dist/agent/tools/inspectorTool.js +428 -0
  47. package/dist/agent/tools/inspectorTool.js.map +1 -0
  48. package/dist/agent/toolset.d.ts +16 -6
  49. package/dist/agent/toolset.js +17 -7
  50. package/dist/agent/toolset.js.map +1 -1
  51. package/dist/agent/types.d.ts +2 -0
  52. package/dist/components/Application.d.ts +73 -18
  53. package/dist/components/Application.js +475 -64
  54. package/dist/components/Application.js.map +1 -1
  55. package/dist/components/OptionsWatcher.d.ts +1 -1
  56. package/dist/components/OptionsWatcher.js +65 -2
  57. package/dist/components/OptionsWatcher.js.map +1 -1
  58. package/dist/components/Scope.d.ts +8 -6
  59. package/dist/components/Scope.js +22 -6
  60. package/dist/components/Scope.js.map +1 -1
  61. package/dist/components/componentLoader.js +5 -1
  62. package/dist/components/componentLoader.js.map +1 -1
  63. package/dist/components/componentSecrets.d.ts +33 -7
  64. package/dist/components/componentSecrets.js +473 -59
  65. package/dist/components/componentSecrets.js.map +1 -1
  66. package/dist/components/deploymentRecorder.d.ts +2 -2
  67. package/dist/components/deploymentRecorder.js +1 -1
  68. package/dist/components/deploymentRecorder.js.map +1 -1
  69. package/dist/components/gitCredentialHelper.d.ts +1 -0
  70. package/dist/components/gitCredentialHelper.js +113 -0
  71. package/dist/components/gitCredentialHelper.js.map +1 -0
  72. package/dist/components/gitCredentialServer.d.ts +33 -0
  73. package/dist/components/gitCredentialServer.js +236 -0
  74. package/dist/components/gitCredentialServer.js.map +1 -0
  75. package/dist/components/mcp/tools/application.d.ts +19 -0
  76. package/dist/components/mcp/tools/application.js +180 -22
  77. package/dist/components/mcp/tools/application.js.map +1 -1
  78. package/dist/components/mcp/tools/operations.d.ts +13 -0
  79. package/dist/components/mcp/tools/operations.js +1 -0
  80. package/dist/components/mcp/tools/operations.js.map +1 -1
  81. package/dist/components/mcp/tools/schemas/derive.js +6 -2
  82. package/dist/components/mcp/tools/schemas/derive.js.map +1 -1
  83. package/dist/components/operations.js +23 -21
  84. package/dist/components/operations.js.map +1 -1
  85. package/dist/components/operationsValidation.js +84 -21
  86. package/dist/components/operationsValidation.js.map +1 -1
  87. package/dist/components/secretOperations.d.ts +33 -11
  88. package/dist/components/secretOperations.js +90 -26
  89. package/dist/components/secretOperations.js.map +1 -1
  90. package/dist/config/harperConfigEnvVars.d.ts +21 -0
  91. package/dist/config/harperConfigEnvVars.js +95 -5
  92. package/dist/config/harperConfigEnvVars.js.map +1 -1
  93. package/dist/index.d.ts +4 -0
  94. package/dist/index.js +14 -1
  95. package/dist/index.js.map +1 -1
  96. package/dist/resources/Resource.d.ts +19 -0
  97. package/dist/resources/Resource.js +93 -2
  98. package/dist/resources/Resource.js.map +1 -1
  99. package/dist/resources/Table.d.ts +9 -0
  100. package/dist/resources/Table.js +256 -50
  101. package/dist/resources/Table.js.map +1 -1
  102. package/dist/resources/analytics/read.js +28 -25
  103. package/dist/resources/analytics/read.js.map +1 -1
  104. package/dist/resources/defineResource.d.ts +180 -0
  105. package/dist/resources/defineResource.js +505 -0
  106. package/dist/resources/defineResource.js.map +1 -0
  107. package/dist/resources/defineTable.d.ts +221 -0
  108. package/dist/resources/defineTable.js +227 -0
  109. package/dist/resources/defineTable.js.map +1 -0
  110. package/dist/resources/jsonSchemaTypes.d.ts +2 -0
  111. package/dist/resources/jsonSchemaTypes.js +12 -4
  112. package/dist/resources/jsonSchemaTypes.js.map +1 -1
  113. package/dist/resources/openApi.js +69 -13
  114. package/dist/resources/openApi.js.map +1 -1
  115. package/dist/resources/search.js +6 -8
  116. package/dist/resources/search.js.map +1 -1
  117. package/dist/server/graphqlQuerying.js +4 -2
  118. package/dist/server/graphqlQuerying.js.map +1 -1
  119. package/dist/server/http.d.ts +12 -0
  120. package/dist/server/http.js +35 -15
  121. package/dist/server/http.js.map +1 -1
  122. package/dist/server/serverHelpers/serverUtilities.js +8 -5
  123. package/dist/server/serverHelpers/serverUtilities.js.map +1 -1
  124. package/dist/server/serverHelpers/uwsServer.js +23 -0
  125. package/dist/server/serverHelpers/uwsServer.js.map +1 -1
  126. package/dist/utility/errors/hdbError.d.ts +21 -0
  127. package/dist/utility/errors/hdbError.js +23 -1
  128. package/dist/utility/errors/hdbError.js.map +1 -1
  129. package/dist/validation/configValidator.js +22 -6
  130. package/dist/validation/configValidator.js.map +1 -1
  131. package/index.ts +33 -0
  132. package/npm-shrinkwrap.json +9197 -15236
  133. package/package.json +5 -3
  134. package/resources/DESIGN.md +42 -15
  135. package/resources/Resource.ts +99 -2
  136. package/resources/Table.ts +275 -71
  137. package/resources/analytics/read.ts +30 -25
  138. package/resources/defineResource.ts +651 -0
  139. package/resources/defineTable.ts +407 -0
  140. package/resources/jsonSchemaTypes.ts +12 -4
  141. package/resources/openApi.ts +68 -16
  142. package/resources/search.ts +5 -8
  143. package/server/graphqlQuerying.ts +4 -2
  144. package/server/http.ts +40 -16
  145. package/server/serverHelpers/serverUtilities.ts +19 -6
  146. package/server/serverHelpers/uwsServer.ts +24 -0
  147. package/studio/web/assets/{Chat-D4FIaBu7.js → Chat-CTjtL8Z4.js} +2 -2
  148. package/studio/web/assets/{Chat-D4FIaBu7.js.map → Chat-CTjtL8Z4.js.map} +1 -1
  149. package/studio/web/assets/{FloatingChat-DGUC3L4r.js → FloatingChat-CafHR4Ur.js} +4 -4
  150. package/studio/web/assets/{FloatingChat-DGUC3L4r.js.map → FloatingChat-CafHR4Ur.js.map} +1 -1
  151. package/studio/web/assets/{applications-D8Am9ikM.js → applications-Buh_q0Vj.js} +2 -2
  152. package/studio/web/assets/{applications-D8Am9ikM.js.map → applications-Buh_q0Vj.js.map} +1 -1
  153. package/studio/web/assets/{index-HGL4WHdb.js → index-0hXeECkS.js} +6 -6
  154. package/studio/web/assets/{index-HGL4WHdb.js.map → index-0hXeECkS.js.map} +1 -1
  155. package/studio/web/assets/{index.lazy-Dk5j1AoQ.js → index.lazy-B00B7VBT.js} +4 -4
  156. package/studio/web/assets/{index.lazy-Dk5j1AoQ.js.map → index.lazy-B00B7VBT.js.map} +1 -1
  157. package/studio/web/assets/{profile-7bu_CF1f.js → profile-Cg2wwYPn.js} +2 -2
  158. package/studio/web/assets/{profile-7bu_CF1f.js.map → profile-Cg2wwYPn.js.map} +1 -1
  159. package/studio/web/assets/{setComponentFile-MfYm9F94.js → setComponentFile-DCaDIvyB.js} +2 -2
  160. package/studio/web/assets/{setComponentFile-MfYm9F94.js.map → setComponentFile-DCaDIvyB.js.map} +1 -1
  161. package/studio/web/assets/{setup-SKA4UhdE.js → setup-CAVcAQjK.js} +2 -2
  162. package/studio/web/assets/{setup-SKA4UhdE.js.map → setup-CAVcAQjK.js.map} +1 -1
  163. package/studio/web/assets/{status-DetVLUxR.js → status-BRXorNdD.js} +2 -2
  164. package/studio/web/assets/{status-DetVLUxR.js.map → status-BRXorNdD.js.map} +1 -1
  165. package/studio/web/assets/{swagger-ui-react-DLeYARY2.js → swagger-ui-react-Dy1D62vO.js} +2 -2
  166. package/studio/web/assets/{swagger-ui-react-DLeYARY2.js.map → swagger-ui-react-Dy1D62vO.js.map} +1 -1
  167. package/studio/web/assets/{tsMode-DGO-jIF7.js → tsMode-A8gbL74v.js} +2 -2
  168. package/studio/web/assets/{tsMode-DGO-jIF7.js.map → tsMode-A8gbL74v.js.map} +1 -1
  169. package/studio/web/assets/{useEntityRestURL-4M2i6bo4.js → useEntityRestURL-cDodrVcQ.js} +2 -2
  170. package/studio/web/assets/{useEntityRestURL-4M2i6bo4.js.map → useEntityRestURL-cDodrVcQ.js.map} +1 -1
  171. package/studio/web/index.html +1 -1
  172. package/utility/errors/hdbError.ts +32 -0
  173. package/validation/configValidator.ts +23 -6
@@ -34,8 +34,12 @@
34
34
  * rather than guessing. The recommended idiom is a module-top-level destructure
35
35
  * (`const { MY_KEY } = secrets;`), where binding is exact in every mode.
36
36
  *
37
- * Late custody / changed secrets heal on restart or component reload (each load cycle re-reads the
38
- * store) there is no live re-materialization.
37
+ * Liveness (#1776): a change watcher on the store keeps the SCOPED tier live the per-component
38
+ * accessor (`secrets.NAME`) and `secrets.subscribe(name)` reflect a later set_secret/grant/revoke/delete
39
+ * without a reload. The GLOBAL (`process.env`) tier stays reload-only: a live change never re-mutates
40
+ * `process.env` under already-running code (child processes have inherited it; arbitrary code has cached
41
+ * it), so materialized env values still heal only on restart or component reload. Late custody likewise
42
+ * heals on reload (each load cycle re-reads and re-decrypts the store).
39
43
  */
40
44
  import { AsyncLocalStorage } from 'node:async_hooks';
41
45
  import { databases } from '../resources/databases.ts';
@@ -47,8 +51,19 @@ import { _assignPackageExport } from '../globals.js';
47
51
 
48
52
  const SECRET_TABLE = SYSTEM_TABLE_NAMES.SECRET_TABLE_NAME;
49
53
 
50
- /** A per-component secrets view: read-only name→value map exposed as `import { secrets } from 'harper'`. */
51
- export type SecretsView = Readonly<Record<string, string>>;
54
+ /**
55
+ * Subscribe to a secret's current value and any subsequent changes (#1776). Yields the current value
56
+ * first, then on each change; `undefined` means the component has no access right now (never granted,
57
+ * revoked, or deleted) — the same thing `secrets[name]` reads as. The stream stays open across that,
58
+ * so a re-grant/re-add resumes on the same iterator.
59
+ */
60
+ export type SecretSubscribe = (name: string) => AsyncIterableIterator<string | undefined>;
61
+
62
+ /**
63
+ * A per-component secrets view: read-only name→value map exposed as `import { secrets } from 'harper'`,
64
+ * plus the reserved, non-enumerable `subscribe(name)` method for live change subscription.
65
+ */
66
+ export type SecretsView = Readonly<Record<string, string>> & { readonly subscribe: SecretSubscribe };
52
67
 
53
68
  export type UnsatisfiedReason = 'missing' | 'ungranted' | 'custody-unavailable';
54
69
 
@@ -89,7 +104,40 @@ const ownedEnvKeys = new Set<string>();
89
104
  // Per-component registries, rebuilt as each component's env block is processed.
90
105
  const declaredEnvNames = new Map<string, Set<string>>();
91
106
  const unsatisfiedEnv = new Map<string, UnsatisfiedDeclaration[]>();
107
+ // Cached LIVE per-component view proxies (stable per component; they resolve values on each read, so
108
+ // they never need invalidating on a secret change — only on reset).
92
109
  const accessorCache = new Map<string, SecretsView>();
110
+ // Components already warned about a secret named `subscribe` shadowing the reserved method.
111
+ const warnedSubscribeShadow = new Set<string>();
112
+
113
+ /**
114
+ * Build the in-memory state for one hdb_secret row: normalize grants, read the tier flag, and
115
+ * decrypt the envelope with this node's custody. The single source of truth for turning a stored
116
+ * record into a {grants, processEnv, value?, failure?} — shared by the load-cycle scan
117
+ * (doMaterializeGlobalSecrets) and the live change watcher (dispatchSecretChange), so both paths
118
+ * apply identical decrypt/failure semantics.
119
+ */
120
+ function rowStateFromRecord(record: any): SecretRowState {
121
+ const state: SecretRowState = {
122
+ grants: Array.isArray(record.grants) ? [...new Set(record.grants as string[])] : [],
123
+ processEnv: record.processEnv === true,
124
+ };
125
+ if (typeof record.envelope === 'string') {
126
+ const decryptor = getSecretDecryptor();
127
+ if (!decryptor) {
128
+ state.failure = 'no secrets custody is registered on this node';
129
+ } else {
130
+ try {
131
+ state.value = decryptor(record.envelope);
132
+ } catch (error) {
133
+ state.failure = `decrypt failed: ${(error as Error).message}`;
134
+ }
135
+ }
136
+ } else {
137
+ state.failure = 'stored row has no envelope';
138
+ }
139
+ return state;
140
+ }
93
141
 
94
142
  /**
95
143
  * Read the hdb_secret store, decrypt what this node's custody allows, materialize the global tier
@@ -118,48 +166,74 @@ export function materializeGlobalSecrets(): Promise<void> {
118
166
  }
119
167
  let materializeInFlight: Promise<void> | undefined;
120
168
 
169
+ // Bump on every applied live change, so a full rescan can detect that its stable-snapshot view raced
170
+ // a newer watcher update and retry rather than clobber it (see scanIntoSecretRows).
171
+ let secretEventSeq = 0;
172
+ const SCAN_RETRY_LIMIT = 5;
173
+
174
+ /**
175
+ * Rebuild `secretRows` (the live snapshot behind the scoped accessor and subscriptions) from a full
176
+ * table scan, then re-deliver current values to every open stream. `table.search` uses a stable
177
+ * snapshot, so a scan that started before a revoke could otherwise complete afterward and re-authorize
178
+ * the just-revoked secret; to prevent that we retry the scan when a live change was applied while it
179
+ * ran, and the final assignment is synchronous (no await) so no event can interleave between the guard
180
+ * check and the assignment. Does NOT touch process.env — callers that must (re)materialize the global
181
+ * tier do that separately (it stays reload-only).
182
+ */
183
+ async function scanIntoSecretRows(table: any): Promise<Map<string, SecretRowState>> {
184
+ let rows: Map<string, SecretRowState>;
185
+ let before: number;
186
+ let attempts = 0;
187
+ do {
188
+ before = secretEventSeq;
189
+ rows = new Map();
190
+ for await (const record of table.search([])) {
191
+ const name = record.name;
192
+ if (typeof name !== 'string' || !name) continue;
193
+ rows.set(name, rowStateFromRecord(record));
194
+ }
195
+ } while (before !== secretEventSeq && ++attempts < SCAN_RETRY_LIMIT);
196
+ if (before !== secretEventSeq) {
197
+ logger.warn(
198
+ 'Secrets store changed repeatedly during a rescan; applying the latest scan (heals on the next change)'
199
+ );
200
+ }
201
+ secretRows = rows;
202
+ storeAvailable = true;
203
+ redispatchAllStreams();
204
+ return rows;
205
+ }
206
+
121
207
  async function doMaterializeGlobalSecrets(): Promise<void> {
122
208
  const table = (databases as { system?: Record<string, any> }).system?.[SECRET_TABLE];
123
209
  if (!table) {
124
210
  storeAvailable = false;
125
211
  secretRows = new Map();
126
212
  accessorCache.clear();
213
+ redispatchAllStreams(); // reflect the store going away to any open streams
127
214
  logger.debug?.(
128
215
  `Secrets store not initialized (system.${SECRET_TABLE} missing); component env declarations can only be satisfied from process.env`
129
216
  );
130
217
  return;
131
218
  }
132
- const rows = new Map<string, SecretRowState>();
133
- const decryptor = getSecretDecryptor();
219
+ // Start the watcher BEFORE the scan so a change committed during the scan bumps secretEventSeq and
220
+ // forces a retry (closing the scan→attach gap where a deletion would otherwise be lost). Non-fatal:
221
+ // a subscribe failure leaves the node bootable with reload-only secrets.
134
222
  try {
135
- for await (const row of table.search([])) {
136
- const name = row.name;
137
- if (typeof name !== 'string' || !name) continue;
138
- const state: SecretRowState = {
139
- grants: Array.isArray(row.grants) ? [...new Set(row.grants as string[])] : [],
140
- processEnv: row.processEnv === true,
141
- };
142
- if (typeof row.envelope === 'string') {
143
- if (!decryptor) {
144
- state.failure = 'no secrets custody is registered on this node';
145
- } else {
146
- try {
147
- state.value = decryptor(row.envelope);
148
- } catch (error) {
149
- state.failure = `decrypt failed: ${(error as Error).message}`;
150
- }
151
- }
152
- } else {
153
- state.failure = 'stored row has no envelope';
154
- }
155
- rows.set(name, state);
156
- }
223
+ await ensureSecretWatcher(table);
224
+ } catch (error) {
225
+ logger.warn(
226
+ `Could not start the live secrets change watcher (secrets will be reload-only): ${(error as Error).message}`
227
+ );
228
+ }
229
+ let rows: Map<string, SecretRowState>;
230
+ try {
231
+ rows = await scanIntoSecretRows(table);
157
232
  } catch (error) {
158
233
  // Leave the previous snapshot (and process.env) untouched rather than acting on a partial read.
159
234
  logger.error(`Failed to read the secrets store (system.${SECRET_TABLE}): ${(error as Error).message}`);
160
235
  return;
161
236
  }
162
- storeAvailable = true;
163
237
 
164
238
  for (const [name, state] of rows) {
165
239
  if (!state.processEnv) {
@@ -192,8 +266,6 @@ async function doMaterializeGlobalSecrets(): Promise<void> {
192
266
  ownedEnvKeys.delete(name);
193
267
  }
194
268
  }
195
- secretRows = rows;
196
- accessorCache.clear();
197
269
  }
198
270
 
199
271
  function parseDeclaration(componentName: string, name: string, spec: unknown): EnvDeclaration {
@@ -337,33 +409,407 @@ export function getUnsatisfiedEnv(componentName: string): UnsatisfiedDeclaration
337
409
  return unsatisfiedEnv.get(componentName) ?? [];
338
410
  }
339
411
 
412
+ // ── Live secret change subscription (#1776) ────────────────────────────────────────────────────
413
+ //
414
+ // `secrets.subscribe(name)` returns an async iterable that yields the secret's current effective
415
+ // value (if the component has access now) and then a new value on every change — so an app can
416
+ // hot-swap an API key without a restart. It is a decrypted, grant-filtered PROJECTION over the
417
+ // replicated `system.hdb_secret` table's own subscription: a `set_secret`/`grant_secret` on ANY
418
+ // node replicates the row, the one shared table watcher observes the commit, and the change fans
419
+ // out to the subscribers of that name. Change authority is re-evaluated per event through the SAME
420
+ // resolver as the read accessor (continuous re-authorization, the property #1414 enforces), so a
421
+ // `revoke_secret`/`delete_secret` that removes the component's access hands the stream `undefined`
422
+ // (and retains no plaintext) rather than the last value — the app can never keep receiving a secret
423
+ // it has lost rights to. The stream stays open across that, so a later re-grant/re-add resumes on it.
424
+ //
425
+ // Tier semantics: the SCOPED tier is fully live (value resolved from the freshly decrypted row).
426
+ // The global (processEnv) tier stays reload-only for `process.env` itself — a live change never
427
+ // silently re-mutates process.env under already-running code — but a subscriber to a declared
428
+ // global name still receives the fresh decrypted value through this channel.
429
+
340
430
  /**
341
- * The secrets view for a component: scoped-tier rows granted to it, plus its DECLARED names
342
- * resolved from process.env (global-tier materialized values, env literals, or real env vars).
343
- * The superset lets app code use `secrets.FOO` uniformly, and lets ops later tighten a secret from
344
- * global to granted without breaking the app. Values are decrypted eagerly at load; the object is
345
- * frozen and enumerable (`Object.keys`, spread).
431
+ * A single (component, secret-name) live subscription: a minimal push async-iterator that yields the
432
+ * secret's current effective value and then a new value on every change. A value of `undefined` means
433
+ * the component has no access right now (never granted, revoked, or the secret was deleted) the same
434
+ * thing `secrets[name]` reads as. The stream stays open across that transition, so a delete-then-re-add
435
+ * or revoke-then-re-grant resumes on the SAME iterator with no re-subscribe. It ends only when the
436
+ * consumer breaks out, the component is unloaded, or the store is unavailable.
346
437
  */
347
- export function getSecretsForComponent(componentName: string): SecretsView {
348
- let view = accessorCache.get(componentName);
349
- if (view) return view;
350
- // Null prototype so inherited Object.prototype members (toString, hasOwnProperty, constructor)
351
- // can never masquerade as secret values under dynamic access like `secrets[key]`.
352
- const entries: Record<string, string> = Object.create(null);
353
- const declared = declaredEnvNames.get(componentName);
354
- if (declared) {
355
- for (const name of declared) {
356
- const value = process.env[name];
357
- if (value !== undefined) entries[name] = value;
438
+ type SecretValue = string | undefined;
439
+ class SecretChangeStream implements AsyncIterableIterator<SecretValue> {
440
+ // At most one undelivered value: this is a current-state stream, so a newer value COALESCES over an
441
+ // unconsumed older one. That also means a revoke (deliver `undefined`) drops any queued plaintext a
442
+ // slow consumer had not yet read — no stale secret is handed out after access is lost.
443
+ #queued: { value: SecretValue } | null = null;
444
+ // FIFO of parked next() resolvers, so concurrent next() calls each settle (a single slot would drop
445
+ // all but the last).
446
+ #waiters: Array<(result: IteratorResult<SecretValue>) => void> = [];
447
+ #closed = false;
448
+ /** Whether any value has been delivered yet, so the first (possibly `undefined`) value always emits. */
449
+ #hasEmitted = false;
450
+ /** Last value delivered, so an unrelated row change doesn't re-emit an unchanged value. */
451
+ lastValue: SecretValue;
452
+ readonly componentName: string;
453
+ readonly name: string;
454
+
455
+ constructor(componentName: string, name: string) {
456
+ this.componentName = componentName;
457
+ this.name = name;
458
+ }
459
+
460
+ get closed(): boolean {
461
+ return this.#closed;
462
+ }
463
+
464
+ [Symbol.asyncIterator](): AsyncIterableIterator<SecretValue> {
465
+ return this;
466
+ }
467
+
468
+ next(): Promise<IteratorResult<SecretValue>> {
469
+ if (this.#queued) {
470
+ const { value } = this.#queued;
471
+ this.#queued = null;
472
+ return Promise.resolve({ value, done: false });
358
473
  }
474
+ if (this.#closed) return Promise.resolve({ value: undefined, done: true });
475
+ return new Promise((resolve) => this.#waiters.push(resolve));
359
476
  }
360
- // Scoped rows granted to this component; on a name collision the scoped value wins.
361
- for (const [name, row] of secretRows) {
362
- if (row.value !== undefined && !row.processEnv && row.grants.includes(componentName)) {
363
- entries[name] = row.value;
477
+
478
+ // Consumer breaking out of `for await` — tear down and stop tracking this stream.
479
+ return(): Promise<IteratorResult<SecretValue>> {
480
+ this.#close();
481
+ return Promise.resolve({ value: undefined, done: true });
482
+ }
483
+
484
+ // Producer side: deliver the current effective value (bespoke iterator rather than IterableEventQueue
485
+ // so an `undefined`/empty-string value can't be misread as an empty queue). Dedups so an unrelated
486
+ // row change doesn't re-emit an unchanged value; the first value always emits so a currently-absent
487
+ // secret yields `undefined` up front.
488
+ deliver(value: SecretValue): void {
489
+ if (this.#closed) return;
490
+ if (this.#hasEmitted && value === this.lastValue) return;
491
+ this.#hasEmitted = true;
492
+ this.lastValue = value;
493
+ const waiter = this.#waiters.shift();
494
+ if (waiter) waiter({ value, done: false });
495
+ else this.#queued = { value }; // replace: coalesce to the latest, dropping any stale queued value
496
+ }
497
+
498
+ // Producer side: end the stream (consumer/component gone, or the store is unavailable). Access loss
499
+ // is NOT an end — it is delivered as an `undefined` value via deliver().
500
+ end(): void {
501
+ this.#close();
502
+ }
503
+
504
+ #close(): void {
505
+ if (this.#closed) return;
506
+ this.#closed = true;
507
+ // Drop any queued plaintext and the retained last value so nothing is readable after close.
508
+ this.#queued = null;
509
+ this.lastValue = undefined;
510
+ unregisterStream(this);
511
+ const waiters = this.#waiters;
512
+ this.#waiters = [];
513
+ for (const waiter of waiters) waiter({ value: undefined, done: true });
514
+ }
515
+ }
516
+
517
+ // name → active streams for that name. Keyed by secret name so a change dispatches only to the
518
+ // streams that care, without every stream filtering every commit.
519
+ const secretSubscribers = new Map<string, Set<SecretChangeStream>>();
520
+
521
+ function registerStream(stream: SecretChangeStream): void {
522
+ let streams = secretSubscribers.get(stream.name);
523
+ if (!streams) secretSubscribers.set(stream.name, (streams = new Set()));
524
+ streams.add(stream);
525
+ }
526
+
527
+ function unregisterStream(stream: SecretChangeStream): void {
528
+ const streams = secretSubscribers.get(stream.name);
529
+ if (!streams) return;
530
+ streams.delete(stream);
531
+ if (streams.size === 0) secretSubscribers.delete(stream.name);
532
+ }
533
+
534
+ /**
535
+ * Resolve a component's effective view of one secret from a freshly-decrypted row state — the SAME
536
+ * authority the read accessor applies (scoped rows require a grant; global/absent rows require a
537
+ * declaration), so the subscription can never widen access. Returns the value when available.
538
+ */
539
+ function resolveSubscribedSecret(
540
+ componentName: string,
541
+ name: string,
542
+ row: SecretRowState | undefined
543
+ ): { available: boolean; value?: string } {
544
+ // `subscribe` is the reserved accessor method, never a readable secret (it is shadowed on the view),
545
+ // so it must be uniformly invisible as a value here too — otherwise subscribe('subscribe') would leak
546
+ // the plaintext of a secret literally named `subscribe` that the accessor hides.
547
+ if (name === 'subscribe') return { available: false };
548
+ // Exactly the read accessor's authority (getSecretsForComponent), so a subscription can never
549
+ // widen access beyond a fresh `secrets[name]` read:
550
+ // 1. a scoped row granted to this component and decryptable → its live decrypted value;
551
+ // 2. otherwise, a name this component DECLARED → its process.env value (global-materialized,
552
+ // literal, or a real env var — reload-only, precedence already resolved in process.env).
553
+ // The process.env fallback is gated on declaration: an undeclared, ungranted component must never
554
+ // read a still-materialized global value out of process.env (that would bypass the grant model).
555
+ if (row && !row.processEnv && row.value !== undefined && row.grants.includes(componentName)) {
556
+ return { available: true, value: row.value };
557
+ }
558
+ if (declaredEnvNames.get(componentName)?.has(name)) {
559
+ const envValue = process.env[name];
560
+ if (envValue !== undefined) return { available: true, value: envValue };
561
+ }
562
+ return { available: false };
563
+ }
564
+
565
+ // The one shared subscription to system.hdb_secret. Lazily started on the first subscribe; a
566
+ // single watcher feeds every stream.
567
+ let secretWatcher: Promise<void> | undefined;
568
+ let secretWatcherSubscription: { emit?: (event: string) => void; end?: () => void } | undefined;
569
+
570
+ function restartSecretWatcher(): void {
571
+ // Drop the memo (and any live subscription) so the next ensureSecretWatcher() re-attaches.
572
+ const sub = secretWatcherSubscription;
573
+ secretWatcher = undefined;
574
+ secretWatcherSubscription = undefined;
575
+ sub?.emit?.('close');
576
+ sub?.end?.();
577
+ }
578
+
579
+ function ensureSecretWatcher(table: any): Promise<void> {
580
+ // Reset the memo on failure so a transient table.subscribe() error doesn't permanently disable live
581
+ // secrets — the next caller retries instead of reusing a rejected promise.
582
+ return (secretWatcher ??= (async () => {
583
+ // omitCurrent: the load-cycle scan (scanIntoSecretRows) seeds and re-syncs the snapshot, and the
584
+ // watcher is attached BEFORE that scan, so it need not (and should not) replay current rows here —
585
+ // it only needs subsequent commits, local or replicated.
586
+ secretWatcherSubscription = await table.subscribe({ omitCurrent: true, listener: onSecretEvent });
587
+ })().catch((error) => {
588
+ secretWatcher = undefined;
589
+ throw error;
590
+ }));
591
+ }
592
+
593
+ // Fired for every commit to system.hdb_secret (local or replicated). event.value is the full record
594
+ // on put; null-valued on delete. Keeps `secretRows` (the live view's source) current and fans the
595
+ // change out to any subscribers, WITHOUT touching process.env — the global tier stays reload-only.
596
+ function onSecretEvent(event: any): void {
597
+ try {
598
+ // Table.subscribe surfaces an async retained-state/replay failure by delivering the Error THROUGH
599
+ // the listener (not by rejecting subscribe()), so detect it here and restart the watcher —
600
+ // otherwise the memo stays fulfilled and every future ensureSecretWatcher reuses a dead subscription.
601
+ if (event instanceof Error) {
602
+ logger.warn(`secrets change watcher errored; restarting it: ${event.message}`);
603
+ restartSecretWatcher();
604
+ return;
605
+ }
606
+ // Base-copy replication of a system table delivers a single `{type:'reload', id:null}` marker
607
+ // instead of per-row events (harper-pro#495): the whole table was reseeded, so re-scan the LIVE
608
+ // snapshot to resync adds AND removals (and re-deliver to streams). It does NOT re-materialize
609
+ // process.env — the global tier stays reload-only even under replicated reloads.
610
+ if (event?.type === 'reload') {
611
+ const table = (databases as { system?: Record<string, any> }).system?.[SECRET_TABLE];
612
+ if (table)
613
+ scanIntoSecretRows(table).catch((error) =>
614
+ logger.warn(`secrets store rescan after a reload marker failed: ${(error as Error).message}`)
615
+ );
616
+ return;
617
+ }
618
+ const name = (event?.value?.name ?? event?.id) as unknown;
619
+ if (typeof name !== 'string' || !name) return;
620
+ secretEventSeq++; // let a concurrent rescan know its snapshot is now stale
621
+ const record = event?.type === 'delete' ? null : (event?.value ?? null);
622
+ const row = record ? rowStateFromRecord(record) : undefined;
623
+ if (row) secretRows.set(name, row);
624
+ else secretRows.delete(name);
625
+ // The cached views are live proxies (they read secretRows on each access), so no invalidation is
626
+ // needed here — a fresh `secrets[name]` read already reflects this update.
627
+ dispatchSecretChange(name, row);
628
+ } catch (error) {
629
+ logger.warn(`secrets change subscription failed to dispatch an event: ${(error as Error).message}`);
630
+ }
631
+ }
632
+
633
+ function dispatchSecretChange(name: string, row: SecretRowState | undefined): void {
634
+ const streams = secretSubscribers.get(name);
635
+ if (!streams || streams.size === 0) return;
636
+ for (const stream of [...streams]) {
637
+ // Authority is re-evaluated on EVERY event (continuous re-authorization, the property #1414
638
+ // enforces): a revoke/delete/custody-loss resolves to unavailable → the stream is handed
639
+ // `undefined` (no plaintext retained), not the last value. A later re-grant/re-add resolves to a
640
+ // value again and resumes on the same stream. deliver() dedups unchanged values.
641
+ const { available, value } = resolveSubscribedSecret(stream.componentName, name, row);
642
+ stream.deliver(available ? value : undefined);
643
+ }
644
+ }
645
+
646
+ // Re-deliver the current effective value to every open stream from the present `secretRows` — used
647
+ // after a full rescan (reload marker or load cycle), where individual per-row events were not seen so
648
+ // a removed row's subscribers would otherwise keep their stale value.
649
+ function redispatchAllStreams(): void {
650
+ for (const [name, streams] of secretSubscribers) {
651
+ const row = secretRows.get(name);
652
+ for (const stream of [...streams]) {
653
+ const { available, value } = resolveSubscribedSecret(stream.componentName, name, row);
654
+ stream.deliver(available ? value : undefined);
364
655
  }
365
656
  }
366
- view = Object.freeze(entries);
657
+ }
658
+
659
+ /**
660
+ * `secrets.subscribe(name)` for a component: the current effective value (matching `secrets[name]`)
661
+ * then a stream of changes. Returned synchronously for direct use in `for await`. The initial value is
662
+ * delivered SYNCHRONOUSLY from the live snapshot before the watcher is (re)started, so no change event
663
+ * can interleave ahead of it and a stale value can never land after a newer one.
664
+ */
665
+ function subscribeSecret(componentName: string, name: string): AsyncIterableIterator<SecretValue> {
666
+ const stream = new SecretChangeStream(componentName, name);
667
+ registerStream(stream);
668
+ // Initial value from the current snapshot (scoped-granted live, or a declared name's process.env
669
+ // value even when the store is unavailable — accessor-equivalent).
670
+ const initial = resolveSubscribedSecret(componentName, name, secretRows.get(name));
671
+ stream.deliver(initial.available ? initial.value : undefined);
672
+ const table = (databases as { system?: Record<string, any> }).system?.[SECRET_TABLE];
673
+ if (table) {
674
+ // Start (or reuse) the one shared watcher so subsequent changes flow to this stream.
675
+ ensureSecretWatcher(table).catch((error) => {
676
+ logger.warn(`secrets.subscribe('${name}'): live change watcher unavailable: ${(error as Error).message}`);
677
+ });
678
+ } else {
679
+ // No store to watch: the stream keeps its initial value and simply won't receive live updates
680
+ // (nothing can change without the store); a reload/restart re-establishes it.
681
+ logger.debug?.(
682
+ `secrets.subscribe('${name}'): secrets store (system.${SECRET_TABLE}) is unavailable; no live updates`
683
+ );
684
+ }
685
+ return stream;
686
+ }
687
+
688
+ /** Close every live subscription owned by a component (component reload/unload). */
689
+ export function closeComponentSubscriptions(componentName: string): void {
690
+ // Collect first, then close: stream.end() → unregisterStream() mutates secretSubscribers, so we must
691
+ // not close while iterating it.
692
+ const toClose: SecretChangeStream[] = [];
693
+ for (const streams of secretSubscribers.values()) {
694
+ for (const stream of streams) if (stream.componentName === componentName) toClose.push(stream);
695
+ }
696
+ for (const stream of toClose) stream.end();
697
+ }
698
+
699
+ // Subscriptions are keyed by component IDENTITY (applicationScope.name), which several concurrently-open
700
+ // Scopes legitimately share — most notably a throwaway deploy-validation Scope loads the SAME directory
701
+ // as the running app and closes in a `finally` before the real restart. Tearing down on any one Scope's
702
+ // close would kill the running app's live streams. So teardown is reference-counted per identity: a Scope
703
+ // retains on construction and releases on close, and streams are ended only when the LAST holder of that
704
+ // identity releases (i.e. the app is truly unloading, not merely a validation load being discarded).
705
+ const subscriptionHolders = new Map<string, number>();
706
+
707
+ /** A Scope of this identity is now open — hold its live secret subscriptions until it releases. */
708
+ export function retainComponentSubscriptions(componentName: string): void {
709
+ subscriptionHolders.set(componentName, (subscriptionHolders.get(componentName) ?? 0) + 1);
710
+ }
711
+
712
+ /** A Scope of this identity closed — end the identity's subscriptions once the last holder is gone. */
713
+ export function releaseComponentSubscriptions(componentName: string): void {
714
+ const remaining = (subscriptionHolders.get(componentName) ?? 0) - 1;
715
+ if (remaining > 0) {
716
+ subscriptionHolders.set(componentName, remaining);
717
+ } else {
718
+ subscriptionHolders.delete(componentName);
719
+ closeComponentSubscriptions(componentName);
720
+ }
721
+ }
722
+
723
+ /** The component's current effective value for one name (scoped-granted live, else declared→process.env). */
724
+ function currentSecretValue(componentName: string, name: string): string | undefined {
725
+ const { available, value } = resolveSubscribedSecret(componentName, name, secretRows.get(name));
726
+ return available ? value : undefined;
727
+ }
728
+
729
+ /**
730
+ * Whether a secret named exactly `name` WOULD be readable to the component if it weren't reserved —
731
+ * used only to detect (and warn about) a secret named `subscribe` shadowed by the reserved method.
732
+ */
733
+ function hasRawSecretNamed(componentName: string, name: string): boolean {
734
+ const row = secretRows.get(name);
735
+ if (row && !row.processEnv && row.value !== undefined && row.grants.includes(componentName)) return true;
736
+ return (declaredEnvNames.get(componentName)?.has(name) ?? false) && process.env[name] !== undefined;
737
+ }
738
+
739
+ /** The names currently visible to a component (granted scoped rows + declared names present in env). */
740
+ function currentSecretNames(componentName: string): string[] {
741
+ const names = new Set<string>();
742
+ const declared = declaredEnvNames.get(componentName);
743
+ if (declared) for (const name of declared) if (process.env[name] !== undefined) names.add(name);
744
+ for (const [name, row] of secretRows) {
745
+ if (row.value !== undefined && !row.processEnv && row.grants.includes(componentName)) names.add(name);
746
+ }
747
+ names.delete('subscribe'); // reserved for the method — never surfaced as a value key
748
+ return [...names];
749
+ }
750
+
751
+ /**
752
+ * The secrets view for a component: scoped-tier rows granted to it, plus its DECLARED names resolved
753
+ * from process.env (global-tier materialized values, env literals, or real env vars). The superset
754
+ * lets app code use `secrets.FOO` uniformly, and lets ops later tighten a secret from global to
755
+ * granted without breaking the app.
756
+ *
757
+ * It is a LIVE view (#1776): a Proxy that resolves each read from the current snapshot, so a scoped
758
+ * secret's rotation/revoke is reflected by a fresh `secrets.FOO` read under every loader — including a
759
+ * view captured once by the vm/compartment loader or held as `const v = scope.secrets` (only a
760
+ * value destructured out, `const { FOO } = secrets`, is a point-in-time copy). Declared global names
761
+ * still resolve from `process.env` (reload-only). Read-only and null-prototype, so Object.prototype
762
+ * members can't masquerade as secrets and the view can't be mutated. `subscribe` is a reserved,
763
+ * non-enumerable method (skipped by Object.keys/spread); a secret literally named `subscribe` is
764
+ * shadowed by it — logged once when observed.
765
+ */
766
+ export function getSecretsForComponent(componentName: string): SecretsView {
767
+ let view = accessorCache.get(componentName);
768
+ if (view) return view;
769
+ const subscribe: SecretSubscribe = (name: string) => subscribeSecret(componentName, name);
770
+ view = new Proxy(Object.create(null) as Record<string, string>, {
771
+ get(_target, property) {
772
+ if (property === 'subscribe') {
773
+ // Advisory: warn once per component if a real secret named `subscribe` is shadowed by the
774
+ // reserved method (it is then unreadable via dot/bracket access — an accepted #1776 tradeoff).
775
+ if (!warnedSubscribeShadow.has(componentName)) {
776
+ warnedSubscribeShadow.add(componentName);
777
+ if (hasRawSecretNamed(componentName, 'subscribe')) {
778
+ logger.warn(
779
+ `Component '${componentName}' has a secret named 'subscribe'; it is shadowed by the reserved secrets.subscribe() method and is not readable through the secrets accessor`
780
+ );
781
+ }
782
+ }
783
+ return subscribe;
784
+ }
785
+ if (typeof property === 'symbol') return undefined;
786
+ return currentSecretValue(componentName, property);
787
+ },
788
+ has(_target, property) {
789
+ if (property === 'subscribe') return true;
790
+ if (typeof property === 'symbol') return false;
791
+ return currentSecretValue(componentName, property) !== undefined;
792
+ },
793
+ ownKeys() {
794
+ // Include `subscribe` so own-property reflection is consistent (getOwnPropertyDescriptor
795
+ // reports it); it is non-enumerable, so spread/Object.keys still skip it.
796
+ return [...currentSecretNames(componentName), 'subscribe'];
797
+ },
798
+ getOwnPropertyDescriptor(_target, property) {
799
+ if (property === 'subscribe') {
800
+ return { value: subscribe, writable: false, enumerable: false, configurable: true };
801
+ }
802
+ if (typeof property === 'symbol') return undefined;
803
+ const value = currentSecretValue(componentName, property);
804
+ if (value === undefined) return undefined;
805
+ return { value, writable: false, enumerable: true, configurable: true };
806
+ },
807
+ set: readOnly,
808
+ defineProperty: readOnly,
809
+ deleteProperty: readOnly,
810
+ preventExtensions: readOnly,
811
+ setPrototypeOf: readOnly,
812
+ }) as unknown as SecretsView;
367
813
  accessorCache.set(componentName, view);
368
814
  return view;
369
815
  }
@@ -407,6 +853,9 @@ function readOnly(): never {
407
853
  export const secrets: SecretsView = new Proxy(
408
854
  {},
409
855
  {
856
+ // `subscribe` resolves like any other own property of the bound view (it is a non-enumerable
857
+ // method there); it therefore reads through here, appears in ownKeys as non-enumerable, and is
858
+ // skipped by spread/Object.keys — no special-casing needed in the traps.
410
859
  get(_target, property) {
411
860
  if (typeof property === 'symbol' || property === 'then') return undefined;
412
861
  return resolveBoundSecrets()[property];
@@ -423,8 +872,7 @@ export const secrets: SecretsView = new Proxy(
423
872
  getOwnPropertyDescriptor(_target, property) {
424
873
  if (typeof property === 'symbol' || componentBinding.getStore() === undefined) return undefined;
425
874
  const descriptor = Object.getOwnPropertyDescriptor(resolveBoundSecrets(), property);
426
- // The views are frozen; report configurable so the proxy invariant against its (extensible,
427
- // empty) target holds.
875
+ // Report configurable so the proxy invariant against this (extensible, empty) target holds.
428
876
  return descriptor && { ...descriptor, configurable: true };
429
877
  },
430
878
  set: readOnly,
@@ -436,7 +884,8 @@ export const secrets: SecretsView = new Proxy(
436
884
  preventExtensions: readOnly,
437
885
  setPrototypeOf: readOnly,
438
886
  }
439
- );
887
+ // The empty target can't carry the reserved `subscribe` member statically; the get trap supplies it.
888
+ ) as unknown as SecretsView;
440
889
  _assignPackageExport('secrets', secrets);
441
890
 
442
891
  /** Reset all module state and retract materialized env values. Intended for tests. */
@@ -448,4 +897,14 @@ export function resetComponentSecrets(): void {
448
897
  declaredEnvNames.clear();
449
898
  unsatisfiedEnv.clear();
450
899
  accessorCache.clear();
900
+ warnedSubscribeShadow.clear();
901
+ const openStreams = [...secretSubscribers.values()].flatMap((streams) => [...streams]);
902
+ secretSubscribers.clear();
903
+ subscriptionHolders.clear();
904
+ for (const stream of openStreams) stream.end();
905
+ // Tear down the shared table watcher too, so a reset doesn't leak an audit-log listener.
906
+ secretWatcherSubscription?.emit?.('close');
907
+ secretWatcherSubscription?.end?.();
908
+ secretWatcher = undefined;
909
+ secretWatcherSubscription = undefined;
451
910
  }