@zackbart/connecta 0.37.0 → 0.38.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (183) hide show
  1. package/CHANGELOG.md +80 -0
  2. package/README.md +2 -2
  3. package/bin/connecta.mjs +1 -39
  4. package/bin/version.mjs +1 -1
  5. package/dist/access-tokens.js +2 -4
  6. package/dist/agent-identity.js +1 -1
  7. package/dist/auth/clerk-sdk/generated.js +2 -1
  8. package/dist/auth/cloudflare-access.d.ts +3 -1
  9. package/dist/auth/cloudflare-access.js +22 -11
  10. package/dist/auth/downstream-oauth.js +195 -270
  11. package/dist/auth/downstream-revocation.js +1 -1
  12. package/dist/auth/oauth-grant.js +11 -0
  13. package/dist/auth/oauth-token-response.js +1 -1
  14. package/dist/auth/vercel-agent.js +2 -1
  15. package/dist/auth-elicitation.js +2 -2
  16. package/dist/branding.d.ts +1 -15
  17. package/dist/branding.js +2 -2
  18. package/dist/bytes.js +42 -0
  19. package/dist/call-admission-context.js +2 -5
  20. package/dist/call-admission.js +5 -0
  21. package/dist/call-recovery.js +0 -3
  22. package/dist/catalog-cache.js +26 -7
  23. package/dist/catalog-fingerprint.js +2 -18
  24. package/dist/catalog-revision.js +9 -0
  25. package/dist/catalog-service.js +51 -20
  26. package/dist/config-defaults.d.ts +1 -0
  27. package/dist/config-defaults.js +1 -0
  28. package/dist/config-schema.d.ts +1 -1
  29. package/dist/config-schema.js +6 -2
  30. package/dist/config.d.ts +41 -14
  31. package/dist/config.js +21 -71
  32. package/dist/connector-caller.js +1 -1
  33. package/dist/connectors/api-connector.js +4 -3
  34. package/dist/connectors/guarded-fetch.d.ts +0 -6
  35. package/dist/connectors/guarded-fetch.js +1 -11
  36. package/dist/connectors/remote-mcp.js +93 -69
  37. package/dist/connectors/resource-uri.js +0 -4
  38. package/dist/credentials.js +3 -0
  39. package/dist/d1.d.ts +0 -89
  40. package/dist/d1.js +0 -34
  41. package/dist/downstream-input.js +2 -2
  42. package/dist/execute.js +161 -116
  43. package/dist/executor-result.js +2 -1
  44. package/dist/executors/quickjs-runtime.js +2 -25
  45. package/dist/executors/quickjs.js +1 -1
  46. package/dist/guest-runtime.js +5 -3
  47. package/dist/guest-types.d.ts +5 -5
  48. package/dist/http/client.js +1 -1
  49. package/dist/http/errors.js +2 -2
  50. package/dist/http/json-schema.js +1 -1
  51. package/dist/http/node-transport.js +0 -1
  52. package/dist/http/oauth.js +1 -1
  53. package/dist/http/paging.js +6 -9
  54. package/dist/http/provider.js +1 -1
  55. package/dist/http/safety.js +1 -1
  56. package/dist/http/services.js +1 -8
  57. package/dist/http/tools.js +1 -7
  58. package/dist/http/vendor.js +7 -7
  59. package/dist/identity.js +2 -6
  60. package/dist/index.js +1 -2
  61. package/dist/invocation-auth.js +2 -1
  62. package/dist/invocation.js +8 -2
  63. package/dist/meta-tools.js +27 -27
  64. package/dist/native-tools.js +2 -1
  65. package/dist/oauth-handoff.js +2 -2
  66. package/dist/operation-vocabulary.js +1 -1
  67. package/dist/operator-record.d.ts +8 -2
  68. package/dist/operator-record.js +18 -50
  69. package/dist/operator-ui/contract.d.ts +4 -2
  70. package/dist/operator-ui/generated.js +3 -3
  71. package/dist/operator-ui/model.d.ts +7 -2
  72. package/dist/program-writes.js +3 -8
  73. package/dist/provider-call.js +0 -7
  74. package/dist/provider.d.ts +1 -1
  75. package/dist/providers/_shared/google/kit.js +1 -6
  76. package/dist/providers/_shared/google/service-account.js +1 -3
  77. package/dist/providers/_shared/google/workspace.d.ts +1 -1
  78. package/dist/providers/_shared/guide.js +3 -0
  79. package/dist/providers/_shared/json.js +12 -0
  80. package/dist/providers/_shared/rest/compact-index.js +1 -1
  81. package/dist/providers/_shared/rest/kit.js +1 -3
  82. package/dist/providers/_shared/rest/operation-index.js +1 -3
  83. package/dist/providers/_shared/rest/value-safety.js +26 -28
  84. package/dist/providers/breeze/api.js +2 -7
  85. package/dist/providers/breeze/index.js +4 -2
  86. package/dist/providers/breeze/tools.js +3 -4
  87. package/dist/providers/ccb/api.js +1 -6
  88. package/dist/providers/ccb/index.js +2 -1
  89. package/dist/providers/ccb/tools.js +1 -6
  90. package/dist/providers/cloudflare/api.js +2 -2
  91. package/dist/providers/cloudflare/index.js +2 -1
  92. package/dist/providers/cloudflare/safety.js +3 -3
  93. package/dist/providers/docs/tools.js +2 -10
  94. package/dist/providers/drive/tools.js +12 -25
  95. package/dist/providers/forms/tools.js +1 -9
  96. package/dist/providers/gmail/tools.js +5 -21
  97. package/dist/providers/overflow/api.js +1 -3
  98. package/dist/providers/overflow/index.js +2 -1
  99. package/dist/providers/overflow/tools.js +3 -7
  100. package/dist/providers/planning-center/api.js +2 -1
  101. package/dist/providers/planning-center/lib/constants.d.ts +1 -1
  102. package/dist/providers/planning-center/lib/json-api.js +1 -9
  103. package/dist/providers/planning-center/tools.js +2 -1
  104. package/dist/providers/sheets/tools.js +1 -9
  105. package/dist/providers/slides/tools.js +4 -13
  106. package/dist/providers/stripe/index.js +2 -1
  107. package/dist/providers/stripe/safety.js +2 -2
  108. package/dist/providers/tithely/api.js +2 -7
  109. package/dist/providers/tithely/index.js +3 -2
  110. package/dist/providers/tithely/tools.js +7 -5
  111. package/dist/providers/vercel/api.js +1 -6
  112. package/dist/providers/vercel/lib/pin.js +1 -1
  113. package/dist/providers/vercel/safety.js +1 -1
  114. package/dist/providers/vercel/tools.js +2 -1
  115. package/dist/redaction-marker.js +2 -2
  116. package/dist/registry.d.ts +4 -1
  117. package/dist/registry.js +35 -22
  118. package/dist/request-principal.js +15 -2
  119. package/dist/request-state.js +2 -2
  120. package/dist/routes/access-tokens.js +1 -1
  121. package/dist/routes/activity-disclosure.js +1 -1
  122. package/dist/routes/connect.js +2 -2
  123. package/dist/routes/credentials.js +1 -1
  124. package/dist/routes/dispatch.js +10 -0
  125. package/dist/routes/mcp.js +23 -21
  126. package/dist/routes/oauth-management.js +1 -1
  127. package/dist/routes/operator.js +53 -7
  128. package/dist/routes/shared.d.ts +8 -8
  129. package/dist/routes/shared.js +22 -15
  130. package/dist/routes/ui-config.js +6 -6
  131. package/dist/routes/ui-data.js +2 -2
  132. package/dist/routes/ui.js +9 -4
  133. package/dist/runtime/call-admission.js +1 -1
  134. package/dist/runtime/call.js +1 -1
  135. package/dist/runtime/settlement.js +1 -1
  136. package/dist/runtime/shared-read.js +2 -15
  137. package/dist/sent-secrets.d.ts +14 -6
  138. package/dist/sent-secrets.js +231 -85
  139. package/dist/server.js +4 -4
  140. package/dist/skills.js +8 -35
  141. package/dist/sqlite.d.ts +0 -19
  142. package/dist/sqlite.js +2 -83
  143. package/dist/storage/keys.js +29 -124
  144. package/dist/storage/sql-native.js +14 -111
  145. package/dist/storage/sql.d.ts +1 -24
  146. package/dist/storage/sql.js +1 -12
  147. package/dist/tool-error.js +4 -0
  148. package/dist/types.d.ts +4 -3
  149. package/dist/ui.js +3 -0
  150. package/dist/url-safety.js +0 -4
  151. package/dist/usage-guide.js +2 -2
  152. package/dist/version.d.ts +1 -1
  153. package/dist/version.js +1 -1
  154. package/dist/worker.js +1 -1
  155. package/documentation/README.md +6 -42
  156. package/documentation/architecture.md +190 -507
  157. package/documentation/auth.md +172 -58
  158. package/documentation/code-mode.md +98 -66
  159. package/documentation/deploying.md +19 -40
  160. package/documentation/integrating.md +8 -19
  161. package/documentation/meta-tools.md +23 -6
  162. package/documentation/migration-0.33.md +5 -5
  163. package/documentation/migration-0.34.md +2 -4
  164. package/documentation/migration-retired-layouts.md +74 -0
  165. package/documentation/operator-tests.md +19 -119
  166. package/documentation/operator-ui.md +46 -8
  167. package/documentation/provider-auth-migration.md +4 -10
  168. package/documentation/provider-migration-0.29.md +4 -15
  169. package/examples/worker/AGENTS.md +4 -6
  170. package/examples/worker/README.md +20 -115
  171. package/guides/connecting-agents.md +7 -30
  172. package/guides/how-it-works.md +2 -2
  173. package/guides/safety-and-trust.md +4 -1
  174. package/package.json +19 -19
  175. package/templates/node/README.md +9 -26
  176. package/templates/node/package.json +1 -1
  177. package/dist/auth/oauth-v2-migration.js +0 -135
  178. package/dist/provider-call-facts.d.ts +0 -115
  179. package/dist/provider-call.d.ts +0 -112
  180. package/dist/storage/d1-copy.js +0 -255
  181. package/dist/storage/sql-schema.d.ts +0 -6
  182. package/examples/worker/kv-to-d1.wrangler.jsonc +0 -26
  183. package/examples/worker/scripts/copy-kv-to-d1.mjs +0 -162
package/dist/skills.js CHANGED
@@ -1,3 +1,4 @@
1
+ import { bytesToHex, utf8Length } from "./bytes.js";
1
2
  import { duringViewEffect } from "./runtime/call.js";
2
3
  import { RequestScope, requestOwner } from "./runtime/request-scope.js";
3
4
  import { upwardStep } from "./upward.js";
@@ -9,7 +10,7 @@ import { runEdge, withDeadlineEffect } from "./runtime/run.js";
9
10
  import { closeScopeOnExit } from "./runtime/connector-scope.js";
10
11
  import { sentSecretsForRequest } from "./sent-secrets.js";
11
12
  const ROUTE = "Choose a route before discovery. One known-address read uses call_tool; one known-address write uses call_destructive_tool. Unknown-address read-only work starts with execute_code to discover, call, and return the answer; use the same route for reduction, multiple or dependent calls, loops, joins, or branches. Keep discovery and calls together when schemas suffice; do not return catalog matches alone. Sample unfamiliar reads.";
12
- const RECOVERY = 'When an auth error includes a handoff, give it to the user or operator. Use authorize_connector for a fresh link. Page resultId notices and format:"paged" handles with connecta.result(handleOrId, { page? }). Fetch skills({ name: "usage" }) only for missing guidance or program repair.';
13
+ const RECOVERY = 'When an auth error includes a handoff, give it to the user or operator. Use authorize_connector for a fresh link. Page resultId notices and format:"paged" handles with connecta.result(handleOrId, { page? }). Missing guidance/repair only: skills tool "usage" or connecta.skill("usage").';
13
14
  /**
14
15
  * The always-loaded MCP `instructions` string. An endpoint is writable or
15
16
  * read-only: a writable one lets programs and call_destructive_tool write,
@@ -328,25 +329,6 @@ const MAX_SKILL_CATALOG_BYTES = 8 * 1024 * 1024;
328
329
  const MAX_SKILLS = 1_024;
329
330
  const encoder = new TextEncoder();
330
331
  const PRIVATE = { resultType: "complete", ttlMs: 0, cacheScope: "private" };
331
- /** Bound metadata before cloning it, without another payload-sized byte array. */
332
- function jsonBytes(value) {
333
- const json = JSON.stringify(value);
334
- let bytes = 0;
335
- for (let i = 0; i < json.length; i++) {
336
- const code = json.charCodeAt(i);
337
- if (code < 0x80)
338
- bytes++;
339
- else if (code < 0x800)
340
- bytes += 2;
341
- else if (code >= 0xd800 && code <= 0xdbff && json.charCodeAt(i + 1) >= 0xdc00 && json.charCodeAt(i + 1) <= 0xdfff) {
342
- bytes += 4;
343
- i++;
344
- }
345
- else
346
- bytes += 3;
347
- }
348
- return bytes;
349
- }
350
332
  function localRecord(name, uri, description, content, aliases) {
351
333
  return { entry: { uri, frontmatter: { name, description }, resources: "dynamic" }, content, aliases };
352
334
  }
@@ -369,12 +351,7 @@ const localRecordsNative = /* @__PURE__ */ registerAuth(localRecords, function (
369
351
  (a.id < b.id ? -1 : a.id > b.id ? 1 : 0)), (connector) => authFlow(function* () {
370
352
  const name = validSkillName(connector.id)
371
353
  ? connector.id
372
- : `connector-${[
373
- ...new Uint8Array(yield* upwardStep(authInvoke(crypto.subtle, "digest", ["SHA-256", encoder.encode(connector.id)]))),
374
- ]
375
- .map((byte) => byte.toString(16).padStart(2, "0"))
376
- .join("")
377
- .slice(0, 48)}`;
354
+ : `connector-${bytesToHex(new Uint8Array(yield* upwardStep(authInvoke(crypto.subtle, "digest", ["SHA-256", encoder.encode(connector.id)])))).slice(0, 48)}`;
378
355
  const description = connectorGuideSummary(connector);
379
356
  const root = `skill://connecta/connectors/${encodeURIComponent(name)}`;
380
357
  const content = `---\n${JSON.stringify({ name, description }, null, 2)}\n---\n\n${connectorGuide(connector)}`;
@@ -474,11 +451,7 @@ const withManifestNative = /* @__PURE__ */ registerAuth(withManifest, function (
474
451
  const bytes = encoder.encode(record.content);
475
452
  if (bytes.length > MAX_SKILL_BYTES)
476
453
  throw new ConnectorCallError("unavailable", "Local skill exceeds the byte bound.");
477
- const digest = [
478
- ...new Uint8Array(yield* upwardStep(authInvoke(crypto.subtle, "digest", ["SHA-256", bytes]))),
479
- ]
480
- .map((byte) => byte.toString(16).padStart(2, "0"))
481
- .join("");
454
+ const digest = bytesToHex(new Uint8Array(yield* upwardStep(authInvoke(crypto.subtle, "digest", ["SHA-256", bytes]))));
482
455
  return {
483
456
  ...record,
484
457
  entry: {
@@ -723,7 +696,7 @@ export class SkillsRegistry {
723
696
  const records = yield* upwardStep(Effect.forEach(yield* upwardStep(self.local), (record) => authCall(withManifest, [record]), {
724
697
  concurrency: "unbounded",
725
698
  }));
726
- let catalogBytes = jsonBytes(records.map((record) => record.entry));
699
+ let catalogBytes = utf8Length(JSON.stringify(records.map((record) => record.entry)));
727
700
  // Bounded sequential connector reads avoid unbounded fan-out and finish
728
701
  // cleanup before a later connector's failure can end the whole listing.
729
702
  for (const connector of self.registry.listConnectors()) {
@@ -736,7 +709,7 @@ export class SkillsRegistry {
736
709
  if (!Array.isArray(entries) || records.length + entries.length > MAX_SKILLS)
737
710
  throw new ConnectorCallError("unavailable", "Downstream skills listing exceeds its bound.");
738
711
  for (const original of entries) {
739
- if (catalogBytes + jsonBytes(original) + 1 > MAX_SKILL_CATALOG_BYTES)
712
+ if (catalogBytes + utf8Length(JSON.stringify(original)) + 1 > MAX_SKILL_CATALOG_BYTES)
740
713
  throw new ConnectorCallError("unavailable", "Skills listing exceeds its byte bound.");
741
714
  const entry = structuredClone(original);
742
715
  validateEntry(entry);
@@ -758,14 +731,14 @@ export class SkillsRegistry {
758
731
  if (records.some((record) => record.entry.uri === uri))
759
732
  throw new ConnectorCallError("unavailable", "Downstream skills listing contains duplicate entries.");
760
733
  const mapped = { ...entry, uri, resources };
761
- catalogBytes += jsonBytes(mapped) + 1;
734
+ catalogBytes += utf8Length(JSON.stringify(mapped)) + 1;
762
735
  if (catalogBytes > MAX_SKILL_CATALOG_BYTES)
763
736
  throw new ConnectorCallError("unavailable", "Skills listing exceeds its byte bound.");
764
737
  records.push({ entry: mapped, aliases: [], connector, files });
765
738
  }
766
739
  }
767
740
  if (records.length > MAX_SKILLS ||
768
- jsonBytes(records.map((record) => record.entry)) > MAX_SKILL_CATALOG_BYTES) {
741
+ utf8Length(JSON.stringify(records.map((record) => record.entry))) > MAX_SKILL_CATALOG_BYTES) {
769
742
  throw new ConnectorCallError("unavailable", "Skills listing exceeds its byte or entry bound.");
770
743
  }
771
744
  // Keep the old investigation guide behind every connector and downstream
package/dist/sqlite.d.ts CHANGED
@@ -27,22 +27,3 @@ export declare function sqliteStorage(database: SqliteDatabase): KVStorage;
27
27
  * older than `retentionDays`.
28
28
  */
29
29
  export declare function sqliteActivityStore(database: SqliteDatabase, options?: SqliteActivityOptions): ActivityStore;
30
- export interface StateFileImport {
31
- /** Entries copied into the database. */
32
- imported: number;
33
- /** Entries left alone because the database already held their key. */
34
- kept: number;
35
- /** Entries whose TTL had already passed. */
36
- expired: number;
37
- }
38
- /**
39
- * One-shot migration from the 0.28 `fileStorage` JSON state file: copy every
40
- * live entry, with its expiry, into the database's key-value table. Run it
41
- * once, with the old deployment stopped, before the first start on SQLite.
42
- *
43
- * Keys the database already holds are kept, so a second run copies nothing
44
- * twice and never overwrites state written since. The file is read, never
45
- * changed; delete it once the deployment is verified. A file that is not a
46
- * state file throws before anything is written.
47
- */
48
- export declare function importStateFile(database: SqliteDatabase, statePath: string, now?: number): StateFileImport;
package/dist/sqlite.js CHANGED
@@ -4,12 +4,11 @@
4
4
  // install. `sqliteStorage` is the deployment's `KVStorage` and
5
5
  // `sqliteActivityStore` its activity history; both create their tables on
6
6
  // first use. Node-only: nothing reachable from the root entry imports this.
7
- import { chmodSync, closeSync, mkdirSync, openSync, readFileSync } from "node:fs";
7
+ import { chmodSync, closeSync, mkdirSync, openSync } from "node:fs";
8
8
  import { dirname, resolve } from "node:path";
9
9
  import { DatabaseSync } from "node:sqlite";
10
10
  import { effectSqlDriver, sqlDriverEdge, sqlSync } from "./storage/sql-driver.js";
11
- import { validateStorageKey } from "./storage/keys.js";
12
- import { KV_SCHEMA, sqlActivityStore, sqlStorage, } from "./storage/sql.js";
11
+ import { sqlActivityStore, sqlStorage, } from "./storage/sql.js";
13
12
  /** Milliseconds a write waits for another process's write lock. */
14
13
  const BUSY_TIMEOUT_MS = 5_000;
15
14
  /**
@@ -100,83 +99,3 @@ export function sqliteStorage(database) {
100
99
  export function sqliteActivityStore(database, options) {
101
100
  return sqlActivityStore(sqliteDriver(database), "sqlite", options);
102
101
  }
103
- /**
104
- * One-shot migration from the 0.28 `fileStorage` JSON state file: copy every
105
- * live entry, with its expiry, into the database's key-value table. Run it
106
- * once, with the old deployment stopped, before the first start on SQLite.
107
- *
108
- * Keys the database already holds are kept, so a second run copies nothing
109
- * twice and never overwrites state written since. The file is read, never
110
- * changed; delete it once the deployment is verified. A file that is not a
111
- * state file throws before anything is written.
112
- */
113
- export function importStateFile(database, statePath, now = Date.now()) {
114
- const text = readFileSync(statePath, "utf8");
115
- // The file holds stored state (tokens, sealed credentials), and these
116
- // errors reach the operator's terminal, so they name the file and an entry's
117
- // position, never a key or the parser's account, which quotes the text.
118
- let loaded;
119
- try {
120
- loaded = JSON.parse(text);
121
- }
122
- catch {
123
- throw new TypeError(`${statePath} is not a connecta state file: it is not valid JSON`);
124
- }
125
- if (loaded === null || typeof loaded !== "object" || Array.isArray(loaded)) {
126
- throw new TypeError(`${statePath} is not a connecta state file`);
127
- }
128
- const entries = Object.entries(loaded);
129
- for (const [index, [key, entry]] of entries.entries()) {
130
- validateStorageKey(key);
131
- const valid = entry !== null &&
132
- typeof entry === "object" &&
133
- typeof entry.value === "string" &&
134
- (!("exp" in entry) || Number.isFinite(entry.exp));
135
- if (!valid) {
136
- throw new TypeError(`${statePath} is not a connecta state file: entry ${index + 1} has no string value`);
137
- }
138
- }
139
- let db;
140
- let transaction = false;
141
- let step = "open the SQLite database";
142
- const result = { imported: 0, kept: 0, expired: 0 };
143
- try {
144
- db = typeof database === "string" ? openSqlite(database) : database;
145
- step = "begin the import transaction";
146
- db.exec("BEGIN IMMEDIATE");
147
- transaction = true;
148
- step = "prepare the key-value table";
149
- for (const statement of KV_SCHEMA)
150
- db.exec(statement);
151
- const insert = db.prepare(`INSERT INTO connecta_kv (key, value, expires_at_ms) VALUES (?, ?, ?)
152
- ON CONFLICT (key) DO NOTHING`);
153
- for (const [index, [key, entry]] of entries.entries()) {
154
- step = `import entry ${index + 1} into the key-value table`;
155
- const { value, exp } = entry;
156
- if (exp !== undefined && exp <= now) {
157
- result.expired += 1;
158
- continue;
159
- }
160
- if (Number(insert.run(key, value, exp === undefined ? null : Math.ceil(exp)).changes) > 0) {
161
- result.imported += 1;
162
- }
163
- else {
164
- result.kept += 1;
165
- }
166
- }
167
- step = "commit the import transaction";
168
- db.exec("COMMIT");
169
- }
170
- catch {
171
- if (transaction) {
172
- try {
173
- db.exec("ROLLBACK");
174
- }
175
- catch {
176
- // Rollback errors can contain stored text too. Report the failed step.
177
- }
178
- }
179
- throw new Error(`State import failed: could not ${step}`);
180
- }
181
- return result;
182
- }
@@ -73,6 +73,19 @@ export const resultKeys = {
73
73
  chunk: (id, index) => validateStorageKey(index === 0 ? `result:${id}` : `result:${id}#${index}`),
74
74
  };
75
75
  // --- root families --------------------------------------------------------
76
+ /** Random key for opaque operator viewer partitions; never leaves storage. */
77
+ export const operatorViewerKeys = {
78
+ family: {
79
+ name: "operator-viewer-key",
80
+ scope: "root",
81
+ prefixes: ["operator-viewer:v1:"],
82
+ version: { number: 1, in: "key" },
83
+ codec: textCodec,
84
+ ttl: { kind: "durable" },
85
+ durable: false,
86
+ },
87
+ signingKey: "operator-viewer:v1:signing",
88
+ };
76
89
  /** A deployment's random HMAC key for bound continuation cursors; never expires or leaves storage. */
77
90
  export const continuationKeys = {
78
91
  family: {
@@ -128,7 +141,7 @@ export const catalogKeys = {
128
141
  name: "catalog",
129
142
  scope: "root",
130
143
  prefixes: ["catalog:"],
131
- // Retired v2/v3 listings are inventoried for migration; no runtime reads them.
144
+ // Retired v2/v3 listings expire on their original TTL; no runtime reads them.
132
145
  version: { number: 3, in: "value" },
133
146
  codec: textCodec,
134
147
  ttl: {
@@ -229,38 +242,6 @@ export const credentialKeys = {
229
242
  },
230
243
  credential: (connectorId, owner) => `${owner ? scopes.principal(owner) : ""}${scopes.connector(connectorId)}credential:v1`,
231
244
  };
232
- /** Seconds a Workers KV → D1 copy's resume point stays usable. */
233
- export const KV_COPY_CURSOR_TTL_SECONDS = 7 * 24 * 60 * 60;
234
- /**
235
- * Where a one-shot Workers KV → D1 copy (`copyKvToD1` in `src/d1.ts`) resumes,
236
- * held in D1 under a random token. Workers KV's own list cursor can spell the
237
- * last key it listed, so the caller holds the token and never the cursor.
238
- */
239
- export const kvCopyKeys = {
240
- family: {
241
- name: "kv-copy",
242
- scope: "root",
243
- prefixes: ["kv-copy:v1:"],
244
- version: { number: 1, in: "key" },
245
- codec: textCodec,
246
- ttl: { kind: "fixed", seconds: KV_COPY_CURSOR_TTL_SECONDS },
247
- durable: false,
248
- },
249
- cursor: (token) => validateStorageKey(`kv-copy:v1:cursor:${token}`),
250
- };
251
- /** Once traffic uses D1, copying stale KV state is refused. */
252
- export const kvCutoverKeys = {
253
- family: {
254
- name: "kv-cutover",
255
- scope: "root",
256
- prefixes: ["kv-cutover:v1:"],
257
- version: { number: 1, in: "key" },
258
- codec: textCodec,
259
- ttl: { kind: "durable" },
260
- durable: true,
261
- },
262
- source: (source) => validateStorageKey(`kv-cutover:v1:${encodeURIComponent(source)}`),
263
- };
264
245
  // --- connector families ---------------------------------------------------
265
246
  /** Seconds a consent's flow record stays claimable, the same as its link. */
266
247
  export const OAUTH_FLOW_TTL_SECONDS = 15 * 60;
@@ -343,38 +324,6 @@ export const oauthRefreshActiveKeys = {
343
324
  prefix: "oauth:refresh-active:",
344
325
  holder: (epoch, holder) => validateStorageKey(`oauth:refresh-active:${epoch}:${holder}`),
345
326
  };
346
- /** The values layout 2 stored, each under its historical key. */
347
- const oauthV2Field = {
348
- client: "oauth:client",
349
- tokens: "oauth:tokens",
350
- pending: "oauth:pending",
351
- verifier: "oauth:verifier",
352
- state: "oauth:state",
353
- discovery: "oauth:discovery",
354
- };
355
- /**
356
- * Layout 2, read once to migrate it into a grant record and then deleted:
357
- * each value at its field key, suffixed `:epoch:<generation>` once a modern
358
- * generation owned it; the active generation at `oauth:generation`; a reset's
359
- * cleanup lineage at `oauth:cleanup:` and `oauth:cleanup-at:`.
360
- */
361
- export const oauthV2Keys = {
362
- family: {
363
- name: "oauth-v2",
364
- scope: "connector",
365
- prefixes: [...Object.values(oauthV2Field), "oauth:generation", "oauth:cleanup:", "oauth:cleanup-at:"],
366
- version: { number: 2, in: "value" },
367
- codec: textCodec,
368
- ttl: { kind: "durable" },
369
- durable: true,
370
- },
371
- field: oauthV2Field,
372
- /** One listing covers every layout 2 key; the reader keeps those above. */
373
- scan: "oauth:",
374
- /** A value's physical key; `epoch` null is the historical unsuffixed name. */
375
- value: (key, epoch) => validateStorageKey(epoch === null ? key : `${key}:epoch:${epoch}`),
376
- generation: "oauth:generation",
377
- };
378
327
  /** Single-use browser links (`used`/`started`/`failed`/`closed`) and MRTR rounds. */
379
328
  export const oauthConnectKeys = {
380
329
  family: {
@@ -417,62 +366,18 @@ export const inputRetryKeys = {
417
366
  },
418
367
  used: (nonce) => validateStorageKey(`input-retry:v1:${nonce}`),
419
368
  };
420
- /**
421
- * Every family, for the overlap and coverage checks and for migrations that
422
- * copy only what a deployment cannot recreate. Keys a custom connector writes
423
- * through its own `ctx.storage` belong to that connector, inside its
424
- * namespace, and are not listed here.
425
- */
426
- export const KEY_FAMILIES = [
427
- continuationKeys.family,
428
- resultKeys.family,
429
- stashLedgerKeys.family,
430
- stashExpiryKeys.family,
431
- catalogKeys.family,
432
- responseCacheKeys.family,
433
- oauthHandoffKeys.family,
434
- accessTokenKeys.family,
435
- agentObservationKeys.family,
436
- credentialKeys.family,
437
- oauthGrantKeys.family,
438
- oauthFlowKeys.family,
439
- oauthRefreshKeys.family,
440
- oauthRefreshSpentKeys.family,
441
- oauthRefreshActiveKeys.family,
442
- oauthV2Keys.family,
443
- oauthConnectKeys.family,
444
- negotiationKeys.family,
445
- inputRetryKeys.family,
446
- kvCopyKeys.family,
447
- kvCutoverKeys.family,
448
- ];
449
- // Principal, connector, and subject segments are hex digests and connector
450
- // ids, so none holds a colon.
451
- const PRINCIPAL_SCOPE = /^principal:[^:]*:/;
452
- const CONNECTOR_SCOPE = /^conn:[^:]*:/;
453
- const PARTITION_SCOPE = /^(?:results:|subject:[^:]*:)/;
454
- /**
455
- * The family a physical key belongs to, for reports that count keys without
456
- * naming them: one-shot migrations report per family, never per key. A key
457
- * no family claims is `connector-owned` when a custom connector wrote it
458
- * through its own `ctx.storage`, and `unclassified` otherwise (a key an older release wrote).
459
- */
460
- export function familyOfKey(key) {
461
- let rest = key.replace(PRINCIPAL_SCOPE, "");
462
- let scope = "root";
463
- for (const [pattern, nested] of [
464
- [CONNECTOR_SCOPE, "connector"],
465
- [PARTITION_SCOPE, "partition"],
466
- ]) {
467
- const match = pattern.exec(rest);
468
- if (match) {
469
- rest = rest.slice(match[0].length);
470
- scope = nested;
471
- break;
472
- }
473
- }
474
- const family = KEY_FAMILIES.find((candidate) => candidate.scope === scope && candidate.prefixes.some((prefix) => rest.startsWith(prefix)));
475
- if (family)
476
- return family.name;
477
- return scope === "connector" ? "connector-owned" : "unclassified";
478
- }
369
+ /** Retired OAuth names are inspected only to report that fresh consent is required. */
370
+ export const retiredOAuthKeys = {
371
+ scan: "oauth:",
372
+ prefixes: [
373
+ "oauth:client",
374
+ "oauth:tokens",
375
+ "oauth:pending",
376
+ "oauth:verifier",
377
+ "oauth:state",
378
+ "oauth:discovery",
379
+ "oauth:generation",
380
+ "oauth:cleanup:",
381
+ "oauth:cleanup-at:",
382
+ ],
383
+ };
@@ -5,9 +5,9 @@
5
5
  // three methods; nothing in this module knows which one it has. It imports no
6
6
  // platform module, so the Worker subpath can carry it.
7
7
  //
8
- // Atomicity. Every compare-and-set is one statement. SQLite executes a
9
- // statement atomically against the database file, and D1 runs one database's
10
- // statements one at a time against its primary, so a comparison and its write
8
+ // Atomicity. Every compare-and-set comparison and mutation share one statement.
9
+ // SQLite executes a statement atomically against the database file, and D1
10
+ // runs one database's statements one at a time against its primary, so a comparison and its write
11
11
  // cannot interleave with another request's: of fifty concurrent claims on an
12
12
  // absent key, exactly one changes a row.
13
13
  //
@@ -77,6 +77,13 @@ export function nativeSqlStorage(driver) {
77
77
  // SQLite and D1 evaluate 'now' in the database, never in a caller isolate.
78
78
  const now = "CAST((julianday('now') - 2440587.5) * 86400000 AS INTEGER)";
79
79
  const live = `(expires_at_ms IS NULL OR expires_at_ms > ${now})`;
80
+ const expirySweep = () => sql(`DELETE FROM connecta_kv WHERE key IN (
81
+ SELECT key FROM connecta_kv WHERE expires_at_ms <= ${now} LIMIT ${SWEEP_ROWS}
82
+ )`);
83
+ // One atomic batch, one round trip, as `set()` pays. The comparison and
84
+ // mutation stay one statement; only its affected-row count decides the
85
+ // result, never the sweep's.
86
+ const runCas = (statement) => Effect.map(driver.batch([expirySweep(), statement]), (changes) => changes[1] ?? 0);
80
87
  const expiryOptions = (opts) => {
81
88
  if (opts?.expiresAtMs !== undefined) {
82
89
  if (!Number.isSafeInteger(opts.expiresAtMs) || opts.ttlSeconds !== undefined) {
@@ -104,9 +111,7 @@ export function nativeSqlStorage(driver) {
104
111
  yield* ensure();
105
112
  const expiry = expiryOptions(opts);
106
113
  yield* driver.batch([
107
- sql(`DELETE FROM connecta_kv WHERE key IN (
108
- SELECT key FROM connecta_kv WHERE expires_at_ms <= ${now} LIMIT ${SWEEP_ROWS}
109
- )`),
114
+ expirySweep(),
110
115
  sql(`INSERT INTO connecta_kv (key, value, expires_at_ms)
111
116
  VALUES (?, ?, ${expiry.expression})
112
117
  ON CONFLICT (key) DO UPDATE SET
@@ -143,8 +148,7 @@ export function nativeSqlStorage(driver) {
143
148
  return (yield* current(key)) === null;
144
149
  }
145
150
  if (next === null) {
146
- return ((yield* driver.run(sql(`DELETE FROM connecta_kv WHERE key = ? AND ${live} AND value = ?`, key, expected))) >
147
- 0);
151
+ return ((yield* runCas(sql(`DELETE FROM connecta_kv WHERE key = ? AND ${live} AND value = ?`, key, expected))) > 0);
148
152
  }
149
153
  let expiry;
150
154
  try {
@@ -160,120 +164,19 @@ export function nativeSqlStorage(driver) {
160
164
  if (expected === null) {
161
165
  // Insert, or take over a row that has expired. A live row makes the
162
166
  // upsert's WHERE false, so nothing changes and the claim is refused.
163
- return ((yield* driver.run(sql(`INSERT INTO connecta_kv (key, value, expires_at_ms)
167
+ return ((yield* runCas(sql(`INSERT INTO connecta_kv (key, value, expires_at_ms)
164
168
  VALUES (?, ?, ${expiry.expression})
165
169
  ON CONFLICT (key) DO UPDATE SET
166
170
  value = excluded.value, expires_at_ms = excluded.expires_at_ms
167
171
  WHERE connecta_kv.expires_at_ms IS NOT NULL
168
172
  AND connecta_kv.expires_at_ms <= ${now}`, key, next, expiry.value))) > 0);
169
173
  }
170
- return ((yield* driver.run(sql(`UPDATE connecta_kv SET value = ?, expires_at_ms = ${expiry.expression}
174
+ return ((yield* runCas(sql(`UPDATE connecta_kv SET value = ?, expires_at_ms = ${expiry.expression}
171
175
  WHERE key = ? AND ${live} AND value = ?`, next, expiry.value, key, expected))) > 0);
172
176
  });
173
177
  },
174
178
  };
175
179
  }
176
- /** Keys one read compares: D1 binds at most 100 parameters, one is `now`. */
177
- const COPY_READ_KEYS = 99;
178
- /** Writes one transaction carries, by count and by UTF-8 bytes. */
179
- const COPY_BATCH_STATEMENTS = 50;
180
- const COPY_BATCH_BYTES = 4 * 1024 * 1024;
181
- /**
182
- * Copy entries into the key-value table, verbatim: same key, same value,
183
- * same absolute expiry. A live row holding a different value is kept unless
184
- * `overwrite` is set, so a second copy of the same entries writes nothing.
185
- * Writes go out in bounded transactions; the outcomes follow `entries`.
186
- */
187
- export function copyIntoSqlEffect(driver, entries, options) {
188
- return Effect.gen(function* () {
189
- const { overwrite, now } = options;
190
- for (const entry of entries)
191
- validateStorageKey(entry.key);
192
- yield* driver.batch(KV_SCHEMA.map((statement) => sql(statement)));
193
- const live = "(expires_at_ms IS NULL OR expires_at_ms > ?)";
194
- const currentValues = (keys) => Effect.gen(function* () {
195
- const values = new Map();
196
- for (let start = 0; start < keys.length; start += COPY_READ_KEYS) {
197
- const chunk = keys.slice(start, start + COPY_READ_KEYS);
198
- const rows = yield* driver.all(sql(`SELECT key, ${textColumn("value")}, expires_at_ms FROM connecta_kv
199
- WHERE key IN (${chunk.map(() => "?").join(", ")}) AND ${live}`, ...chunk, now));
200
- // The message names no key: a copy's errors reach operator output.
201
- for (const row of rows) {
202
- values.set(row.key, {
203
- value: textOf(row.value, row.value_bytes, "a stored value") ?? "",
204
- expiresAtMs: row.expires_at_ms,
205
- });
206
- }
207
- }
208
- return values;
209
- });
210
- const outcomes = Array.from({ length: entries.length });
211
- const writes = [];
212
- const existing = yield* currentValues(entries.map((entry) => entry.key));
213
- for (const [index, entry] of entries.entries()) {
214
- const current = existing.get(entry.key);
215
- if (current?.value === entry.value && current.expiresAtMs === entry.expiresAtMs) {
216
- outcomes[index] = "unchanged";
217
- }
218
- else if (current !== undefined && !overwrite) {
219
- outcomes[index] = "conflict";
220
- }
221
- else {
222
- outcomes[index] = current === undefined ? "copied" : "overwritten";
223
- writes.push({
224
- index,
225
- bytes: new TextEncoder().encode(entry.key).byteLength + new TextEncoder().encode(entry.value).byteLength + 32,
226
- statement: overwrite
227
- ? sql(`INSERT INTO connecta_kv (key, value, expires_at_ms)
228
- VALUES (?, ?, ?)
229
- ON CONFLICT (key) DO UPDATE SET
230
- value = excluded.value, expires_at_ms = excluded.expires_at_ms`, entry.key, entry.value, entry.expiresAtMs)
231
- : // Absent when read; a live row written since is kept, not replaced.
232
- sql(`INSERT INTO connecta_kv (key, value, expires_at_ms)
233
- VALUES (?, ?, ?)
234
- ON CONFLICT (key) DO UPDATE SET
235
- value = excluded.value, expires_at_ms = excluded.expires_at_ms
236
- WHERE connecta_kv.expires_at_ms IS NOT NULL
237
- AND connecta_kv.expires_at_ms <= ?`, entry.key, entry.value, entry.expiresAtMs, now),
238
- });
239
- }
240
- }
241
- const raced = [];
242
- let batch = [];
243
- let batchBytes = 0;
244
- const flush = () => Effect.gen(function* () {
245
- if (batch.length === 0)
246
- return;
247
- const changes = yield* driver.batch(batch.map((write) => write.statement));
248
- for (const [position, write] of batch.entries()) {
249
- if (!(changes[position] > 0))
250
- raced.push(write.index);
251
- }
252
- batch = [];
253
- batchBytes = 0;
254
- });
255
- for (const write of writes) {
256
- if (batch.length > 0 && (batch.length >= COPY_BATCH_STATEMENTS || batchBytes + write.bytes > COPY_BATCH_BYTES)) {
257
- yield* flush();
258
- }
259
- batch.push(write);
260
- batchBytes += write.bytes;
261
- }
262
- yield* flush();
263
- // A row another writer added between the read and the write: report what
264
- // it holds, and leave it.
265
- if (raced.length > 0) {
266
- const after = yield* currentValues(raced.map((index) => entries[index].key));
267
- for (const index of raced) {
268
- const entry = entries[index];
269
- const current = after.get(entry.key);
270
- outcomes[index] =
271
- current?.value === entry.value && current.expiresAtMs === entry.expiresAtMs ? "unchanged" : "conflict";
272
- }
273
- }
274
- return outcomes;
275
- });
276
- }
277
180
  // --- activity -----------------------------------------------------------
278
181
  /**
279
182
  * The activity table, with keyset paging on `(occurred_at_ms, id)`. Its shape
@@ -1,7 +1,5 @@
1
1
  import type { ActivityStore } from "../activity.js";
2
2
  import type { KVStorage } from "../types.js";
3
- /** The 0.28-compatible key-value table and expiry index. */
4
- export { KV_SCHEMA } from "./sql-schema.js";
5
3
  type SqlValue = string | number | null;
6
4
  export interface SqlStatement {
7
5
  readonly sql: string;
@@ -21,18 +19,6 @@ export interface SqlDriver {
21
19
  }
22
20
  /** The two drivers, as `describe()` names them. */
23
21
  export type SqlKind = "d1" | "sqlite";
24
- export interface SqlCopyEntry {
25
- readonly key: string;
26
- readonly value: string;
27
- /** Epoch milliseconds; null never expires. */
28
- readonly expiresAtMs: number | null;
29
- }
30
- /**
31
- * What a copy did with one entry: written where no live row was, left alone
32
- * because the live row already held the value, refused because it held
33
- * another (`conflict`), or replaced under `overwrite` (`overwritten`).
34
- */
35
- export type SqlCopyOutcome = "copied" | "unchanged" | "conflict" | "overwritten";
36
22
  export interface SqlActivityOptions {
37
23
  /**
38
24
  * Days of history kept. Each write removes a bounded batch of older rows,
@@ -48,13 +34,4 @@ export declare function sqlStorage(driver: SqlDriver, kind: SqlKind): KVStorage;
48
34
  * payload to leak.
49
35
  */
50
36
  export declare function sqlActivityStore(driver: SqlDriver, kind: SqlKind, options?: SqlActivityOptions): ActivityStore;
51
- /**
52
- * Copy entries into the key-value table, verbatim: same key, same value,
53
- * same absolute expiry. A live row holding a different value is kept unless
54
- * `overwrite` is set, so a second copy of the same entries writes nothing.
55
- * Writes go out in bounded transactions; the outcomes follow `entries`.
56
- */
57
- export declare function copyIntoSql(driver: SqlDriver, entries: readonly SqlCopyEntry[], options: {
58
- overwrite: boolean;
59
- now: number;
60
- }): Promise<SqlCopyOutcome[]>;
37
+ export {};
@@ -1,9 +1,7 @@
1
1
  import { runEdge } from "../runtime/run.js";
2
2
  import { registerNativeStorage, registerNativeActivity } from "../runtime/native-storage.js";
3
3
  import { nativeSqlDriver } from "./sql-driver.js";
4
- import { nativeSqlStorage, nativeSqlActivityStore, copyIntoSqlEffect } from "./sql-native.js";
5
- /** The 0.28-compatible key-value table and expiry index. */
6
- export { KV_SCHEMA } from "./sql-schema.js";
4
+ import { nativeSqlStorage, nativeSqlActivityStore } from "./sql-native.js";
7
5
  /** `KVStorage` over one `connecta_kv` table. */
8
6
  export function sqlStorage(driver, kind) {
9
7
  const native = nativeSqlStorage(nativeSqlDriver(driver));
@@ -30,12 +28,3 @@ export function sqlActivityStore(driver, kind, options = {}) {
30
28
  list: (options) => runEdge(native.list(options)),
31
29
  }, native);
32
30
  }
33
- /**
34
- * Copy entries into the key-value table, verbatim: same key, same value,
35
- * same absolute expiry. A live row holding a different value is kept unless
36
- * `overwrite` is set, so a second copy of the same entries writes nothing.
37
- * Writes go out in bounded transactions; the outcomes follow `entries`.
38
- */
39
- export function copyIntoSql(driver, entries, options) {
40
- return runEdge(copyIntoSqlEffect(nativeSqlDriver(driver), entries, options));
41
- }
@@ -0,0 +1,4 @@
1
+ export function toolError(error, fields = {}) {
2
+ const structuredContent = { ...fields, error };
3
+ return { content: [{ type: "text", text: JSON.stringify(structuredContent) }], structuredContent, isError: true };
4
+ }