@feltdb/core 0.12.2 → 0.13.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (67) hide show
  1. package/dist/build-identity.json +4 -4
  2. package/dist/cli/commands.js +8 -2
  3. package/dist/cli/workspace-integration.js +8 -0
  4. package/dist/create/build-identity.json +4 -4
  5. package/dist/create/cli.js +46 -37
  6. package/dist/create/create.js +17 -2
  7. package/dist/create/package-versions.js +1 -1
  8. package/dist/create/server-source/Cargo.lock +4 -4
  9. package/dist/create/server-source/Cargo.toml +1 -1
  10. package/dist/create/server-source/crates/feltdb/src/worker_mesh.rs +204 -1
  11. package/dist/create/server-source/crates/feltdb/src/workload.rs +685 -53
  12. package/dist/create/server-source/crates/feltdb/tests/production_readiness_contract.rs +18 -12
  13. package/dist/create/server-source/crates/feltdb/tests/workload_envelope_contract.rs +17 -5
  14. package/dist/create/server-source/crates/feltdb-client/Cargo.toml +1 -1
  15. package/dist/create/server-source/crates/feltdb-server/Cargo.toml +1 -1
  16. package/dist/create/server-source/crates/feltdb-server/src/agent_execution.rs +511 -0
  17. package/dist/create/server-source/crates/feltdb-server/src/auth.rs +28 -0
  18. package/dist/create/server-source/crates/feltdb-server/src/lib.rs +1 -0
  19. package/dist/create/server-source/crates/feltdb-server/src/main.rs +844 -135
  20. package/dist/create/setup-summary.js +40 -0
  21. package/dist/db.d.ts +27 -1
  22. package/dist/db.d.ts.map +1 -1
  23. package/dist/db.js +17 -4
  24. package/dist/development-runtime-bridge.d.ts +20 -0
  25. package/dist/development-runtime-bridge.d.ts.map +1 -1
  26. package/dist/development-runtime-bridge.js +61 -12
  27. package/dist/http-db.d.ts +13 -0
  28. package/dist/http-db.d.ts.map +1 -1
  29. package/dist/http-db.js +33 -4
  30. package/dist/studio/{KeyManagementPanel-D5FMvvU9.js → KeyManagementPanel-skUNnKRw.js} +36 -29
  31. package/dist/studio/app.d.ts +8 -1
  32. package/dist/studio/app.d.ts.map +1 -1
  33. package/dist/studio/components/KeyManagementPanel.d.ts.map +1 -1
  34. package/dist/studio/components/KeyManagementPanel.js +1 -1
  35. package/dist/studio/components/index.js +2 -2
  36. package/dist/studio/{components-B4eKIUN5.js → components-BSGmHfif.js} +6 -6
  37. package/dist/studio/control-plane/Data.d.ts.map +1 -1
  38. package/dist/studio/control-plane/Instance.d.ts +4 -1
  39. package/dist/studio/control-plane/Instance.d.ts.map +1 -1
  40. package/dist/studio/control-plane/Operations.d.ts +4 -1
  41. package/dist/studio/control-plane/Operations.d.ts.map +1 -1
  42. package/dist/studio/control-plane/index.js +2 -2
  43. package/dist/studio/control-plane/primitives.d.ts +16 -0
  44. package/dist/studio/control-plane/primitives.d.ts.map +1 -1
  45. package/dist/studio/{control-plane-fxMcJn74.js → control-plane-CaBA7Rgv.js} +448 -424
  46. package/dist/studio/index.d.ts +1 -0
  47. package/dist/studio/index.d.ts.map +1 -1
  48. package/dist/studio/index.js +384 -247
  49. package/dist/studio/navigation.d.ts +57 -0
  50. package/dist/studio/navigation.d.ts.map +1 -0
  51. package/dist/studio/studio.css +1 -1
  52. package/dist/studio/utils/control-plane-api.d.ts +2 -0
  53. package/dist/studio/utils/control-plane-api.d.ts.map +1 -1
  54. package/dist/studio/utils/control-plane-api.js +5 -1
  55. package/dist/studio-app/assets/{feltdb_wasm-BatwwogY.js → feltdb_wasm-3hDngl1h.js} +1 -1
  56. package/dist/studio-app/assets/feltdb_wasm_bg-CFQT4gCy.wasm +0 -0
  57. package/dist/studio-app/assets/index-Dql1hS5m.js +30 -0
  58. package/dist/studio-app/assets/{index-CiC8VY-a.css → index-vi21oEQa.css} +1 -1
  59. package/dist/studio-app/index.html +2 -2
  60. package/dist/wasm/feltdb_wasm_bg.wasm +0 -0
  61. package/dist/wasm/package.json +1 -1
  62. package/dist/workspace/local-development-authority.d.ts +19 -0
  63. package/dist/workspace/local-development-authority.d.ts.map +1 -1
  64. package/dist/workspace/local-development-authority.js +335 -15
  65. package/package.json +1 -1
  66. package/dist/studio-app/assets/feltdb_wasm_bg-BX0newOr.wasm +0 -0
  67. package/dist/studio-app/assets/index-DoVvbIay.js +0 -30
@@ -1,13 +1,13 @@
1
1
  {
2
2
  "schema": "feltdb.build-identity.v2",
3
3
  "package": "@feltdb/core",
4
- "version": "0.12.2",
4
+ "version": "0.13.0",
5
5
  "git": {
6
- "sha1": "7a632f5b3e79d800ff27359f7f1a7c081dc7cb69",
6
+ "sha1": "b67312f73ea647ffea6f87337303c170d40d541d",
7
7
  "dirty": false
8
8
  },
9
9
  "path_in_vcs": "packages/core",
10
- "source_digest": "sha256:9db516ed9702dde647cd8d9723e1d997be999a39ddebfcb337dc0c1d721c8110",
10
+ "source_digest": "sha256:7435af3423598673fcce76eb77444ee1e160f39f3fd83a0a788517e315bd3444",
11
11
  "inputs": [
12
12
  "Cargo.lock",
13
13
  "Cargo.toml",
@@ -20,7 +20,7 @@
20
20
  "tools/cli",
21
21
  "tools/create-feltdb"
22
22
  ],
23
- "files": 770,
23
+ "files": 776,
24
24
  "build": {
25
25
  "command": "npm run build",
26
26
  "node": "v24.9.0"
@@ -1834,11 +1834,16 @@ async function handleStudio(args, onReady) {
1834
1834
  : process.env.VITE_FELTDB_MANAGED_APPLICATION_ID || managedProject.applicationId || '';
1835
1835
  const environment = args.includes('--environment')
1836
1836
  ? args[args.indexOf('--environment') + 1] || 'production'
1837
- : managedProject.environment || 'production';
1837
+ // A development session's authority is the local environment; asking it
1838
+ // for `production` names an environment the session does not have.
1839
+ : managedProject.environment || (args.includes('--session-id') && runtime !== 'managed' ? 'local' : 'production');
1838
1840
  const appUrl = args.includes('--app-url')
1839
1841
  ? args[args.indexOf('--app-url') + 1]
1840
1842
  : undefined;
1841
1843
  const sessionValue = (name) => args.includes(name) ? args[args.indexOf(name) + 1] : undefined;
1844
+ // The connection parameters Studio is opened with. Served to a page opened
1845
+ // without them (a bookmark, a pasted path) so it reconnects to this session.
1846
+ let studioQuery = '';
1842
1847
  const session = {
1843
1848
  sessionId: sessionValue('--session-id'),
1844
1849
  workspaceId: sessionValue('--workspace-id'),
@@ -1889,7 +1894,7 @@ async function handleStudio(args, onReady) {
1889
1894
  environment: managedProject.environment || 'production',
1890
1895
  publishEndpoint: '/_feltdb/publish',
1891
1896
  } : undefined;
1892
- response.end(JSON.stringify({ token: connectUrl ? process.env.VITE_FELTDB_API_KEY || '' : '', managedPublish, session, sourceSync: { enabled: true, webllmEnabled, previewOnly, token: studioMutationToken, aiModel: publicAiModelConfiguration() } }));
1897
+ response.end(JSON.stringify({ token: connectUrl ? process.env.VITE_FELTDB_API_KEY || '' : '', managedPublish, session, studioQuery, sourceSync: { enabled: true, webllmEnabled, previewOnly, token: studioMutationToken, aiModel: publicAiModelConfiguration() } }));
1893
1898
  return;
1894
1899
  }
1895
1900
  if (pathname === '/_feltdb/contract') {
@@ -2219,6 +2224,7 @@ async function handleStudio(args, onReady) {
2219
2224
  if (session.pairingUrl)
2220
2225
  parameters.set('pairing', session.pairingUrl);
2221
2226
  const query = `?${parameters.toString()}`;
2227
+ studioQuery = parameters.toString();
2222
2228
  const studioUrl = `http://localhost:${port}/${query}`;
2223
2229
  console.log(`FeltDB Studio ready at ${studioUrl}`);
2224
2230
  onReady?.(studioUrl);
@@ -12,6 +12,7 @@ import path from 'path';
12
12
  import { randomBytes } from 'crypto';
13
13
  import http from 'http';
14
14
  import { spawnSync } from 'child_process';
15
+ import { developmentConnectionMatches } from '@feltdb/core';
15
16
  export const DEVELOPMENT_OBSERVATION_HISTORY_LIMIT = 1000;
16
17
  const FELTDB_IGNORE_PATTERN = /^\/?\.feltdb\/?$/;
17
18
  /**
@@ -82,6 +83,12 @@ export function startPairingDiscoveryServer(token, port = 7799, host = '127.0.0.
82
83
  runtimeJson(response, 409, { error: 'RUNTIME_IDENTITY_MISMATCH' });
83
84
  return;
84
85
  }
86
+ // The session identity above is echoed back by every client; what the
87
+ // database is actually connected to is the part worth checking.
88
+ if (body.connection && !developmentConnectionMatches(runtimeSession, body.connection)) {
89
+ runtimeJson(response, 409, { error: 'RUNTIME_CONNECTION_MISMATCH', connection: body.connection });
90
+ return;
91
+ }
85
92
  if (!body.runtimeInstanceId || !body.applicationId) {
86
93
  runtimeJson(response, 400, { error: 'RUNTIME_IDENTITY_REQUIRED' });
87
94
  return;
@@ -97,6 +104,7 @@ export function startPairingDiscoveryServer(token, port = 7799, host = '127.0.0.
97
104
  runtime: runtimeSession.runtime,
98
105
  authorityUrl: runtimeSession.authorityUrl,
99
106
  applicationUrl: body.applicationUrl || runtimeSession.applicationUrl,
107
+ ...(body.connection ? { connection: body.connection } : {}),
100
108
  registeredAt: previous?.registeredAt ?? now,
101
109
  lastSeenAt: now,
102
110
  expiresAt: now + runtimeTtlMs,
@@ -1,13 +1,13 @@
1
1
  {
2
2
  "schema": "feltdb.build-identity.v2",
3
3
  "package": "create-feltdb",
4
- "version": "0.12.2",
4
+ "version": "0.13.0",
5
5
  "git": {
6
- "sha1": "7a632f5b3e79d800ff27359f7f1a7c081dc7cb69",
6
+ "sha1": "b67312f73ea647ffea6f87337303c170d40d541d",
7
7
  "dirty": false
8
8
  },
9
9
  "path_in_vcs": "tools/create-feltdb",
10
- "source_digest": "sha256:9db516ed9702dde647cd8d9723e1d997be999a39ddebfcb337dc0c1d721c8110",
10
+ "source_digest": "sha256:7435af3423598673fcce76eb77444ee1e160f39f3fd83a0a788517e315bd3444",
11
11
  "inputs": [
12
12
  "Cargo.lock",
13
13
  "Cargo.toml",
@@ -20,7 +20,7 @@
20
20
  "tools/cli",
21
21
  "tools/create-feltdb"
22
22
  ],
23
- "files": 770,
23
+ "files": 776,
24
24
  "build": {
25
25
  "command": "npm run build",
26
26
  "node": "v24.9.0"
@@ -11,15 +11,18 @@ import { spawn, spawnSync } from 'child_process';
11
11
  import { createProject } from './create.js';
12
12
  import { FELTDB_PACKAGE_VERSION } from './package-versions.js';
13
13
  import { configureManagedAccount, finalizeManagedProvisioning } from './managed-account.js';
14
+ import { RUNTIME_CHOICES, agentsIncluded, describeSetup } from './setup-summary.js';
14
15
  import { localFeltdbExecutable } from './development-handoff.js';
15
16
  const __dirname = path.dirname(fileURLToPath(import.meta.url));
16
17
  function parseArgs(args) {
17
18
  const options = {
18
19
  runtime: 'browser',
19
20
  framework: 'react',
20
- distributed: true,
21
+ // Distribution and the browser AI builder add services and downloads a
22
+ // first project does not need; each is one flag away.
23
+ distributed: false,
21
24
  agents: true,
22
- webllm: true,
25
+ webllm: false,
23
26
  capabilities: 'search',
24
27
  };
25
28
  for (let i = 0; i < args.length; i++) {
@@ -42,6 +45,9 @@ function parseArgs(args) {
42
45
  case '--framework':
43
46
  options.framework = args[++i];
44
47
  break;
48
+ case '--distributed':
49
+ options.distributed = true;
50
+ break;
45
51
  case '--no-distributed':
46
52
  options.distributed = false;
47
53
  break;
@@ -164,29 +170,28 @@ async function promptForOptions(defaults) {
164
170
  { label: 'Yes', value: true },
165
171
  { label: 'No', value: false },
166
172
  ];
167
- return {
168
- runtime: await select('Where should FeltDB run?', [
169
- { label: 'Browser', value: 'browser', description: 'local-first with durable browser storage' },
170
- { label: 'Node.js', value: 'node', description: 'application server or worker' },
171
- { label: 'Self-hosted', value: 'self-hosted', description: 'dedicated FeltDB server' },
172
- { label: 'Managed', value: 'managed', description: 'FeltDB-hosted persistence, sync, and workloads' },
173
- ], defaults.runtime),
174
- framework: await select('Choose an application framework:', [
175
- { label: 'React', value: 'react' },
176
- { label: 'Vanilla TypeScript', value: 'vanilla' },
177
- ], defaults.framework),
178
- distributed: await select('Enable distributed operation?', yesNo, defaults.distributed),
179
- agents: await select('Include an autonomous agent example?', yesNo, defaults.agents),
180
- webllm: await select('Install WebLLM for private natural-language app building?', [
181
- { label: 'Yes', value: true, description: 'build FeltDB schemas, capabilities, workflows, and UI locally with AI' },
182
- { label: 'No', value: false, description: 'skip the browser model and AI builder' },
183
- ], defaults.webllm),
184
- capabilities: await select('Choose starter capabilities:', [
173
+ const runtime = await select('Where does your application\'s data live?', RUNTIME_CHOICES.map(choice => ({ label: choice.label, value: choice.value, description: choice.summary })), defaults.runtime);
174
+ const framework = await select('Choose an application framework:', [
175
+ { label: 'React', value: 'react', description: 'a complete starter: sign-up, projects, invitations, activity, assistants' },
176
+ { label: 'Vanilla TypeScript', value: 'vanilla', description: 'a minimal page over the same database' },
177
+ ], defaults.framework);
178
+ // The React starter's assistants are part of its tour and need no model, so
179
+ // only the vanilla template asks; its capabilities are chosen here too.
180
+ const agents = framework === 'react' ? true
181
+ : await select('Include an agent example?', yesNo, defaults.agents);
182
+ const capabilities = framework === 'react' ? defaults.capabilities
183
+ : await select('Choose starter capabilities:', [
185
184
  { label: 'Search', value: 'search' },
186
185
  { label: 'Vector search', value: 'vector' },
187
186
  { label: 'Search + vector search', value: 'search,vector' },
188
- ], defaults.capabilities),
189
- };
187
+ ], defaults.capabilities);
188
+ const webllm = await select('Add the private AI app builder (WebLLM)?', [
189
+ { label: 'No', value: false, description: 'add it later with npm install @feltdb/webllm' },
190
+ { label: 'Yes', value: true, description: 'builds schemas and UI with a model that runs in your browser: a download of a few hundred MB on first use, and WebGPU' },
191
+ ], defaults.webllm);
192
+ // Distribution is a deployment decision, not a first-run one: `--distributed`
193
+ // records it, and development always runs one local authority.
194
+ return { runtime, framework, distributed: defaults.distributed, agents, webllm, capabilities };
190
195
  }
191
196
  async function main() {
192
197
  const args = process.argv.slice(2);
@@ -211,9 +216,9 @@ Deployment Target (Explicit):
211
216
  Best for: Client-side apps, offline-first experiences
212
217
  No server required, full offline capability
213
218
 
214
- --server Node.js server runtime with server-side FeltDB authority
215
- Best for: Server-side applications, API servers
216
- Requires production FeltDB instance or self-host
219
+ --server Data lives in a FeltDB authority the application connects to
220
+ Best for: shared state, server-side work, a path to self-hosting
221
+ \`feltdb dev\` runs a local authority; deploy to feltdb-server
217
222
 
218
223
  --self-host Dedicated FeltDB server with Docker Compose
219
224
  Best for: Staging first, then explicit production promotion
@@ -230,11 +235,15 @@ Options:
230
235
  --managed Explicit: Managed FeltDB service
231
236
  --runtime <runtime> Alternative: Set runtime (browser, node, self-hosted, managed)
232
237
  --framework <framework> Choose framework: react or vanilla (default: react)
233
- --no-distributed Disable distributed operation
234
- --no-agents Exclude agent examples
235
- --webllm Include the private natural-language FeltDB app builder (default)
236
- --no-webllm Do not install WebLLM or generate the AI builder
237
- --capabilities <list> Choose capabilities: search, vector, or search,vector
238
+ --distributed Record that the application will be deployed to several
239
+ feltdb-server nodes (development always runs one)
240
+ --no-distributed Do not record distributed deployment (default)
241
+ --no-agents Exclude the agent example (vanilla template; the React
242
+ starter always includes its assistants)
243
+ --webllm Include the private natural-language app builder; downloads
244
+ a model into the browser on first use and needs WebGPU
245
+ --no-webllm Do not install WebLLM (default)
246
+ --capabilities <list> Vanilla template capabilities: search, vector, or search,vector
238
247
  --no-install Skip npm install
239
248
  --no-start Skip automatic feltdb dev startup
240
249
  -y, --yes Use all defaults (non-interactive mode)
@@ -294,13 +303,13 @@ Learn more: https://github.com/rkendel1/feltdb`);
294
303
  console.log('\n═══════════════════════════════════════════════');
295
304
  console.log('📦 DEPLOYMENT TARGET');
296
305
  console.log('═══════════════════════════════════════════════');
297
- console.log(`\nRuntime: ${options.runtime.toUpperCase()}`);
298
- console.log(`Description: ${runtimeDescriptions[options.runtime]}`);
299
- console.log(`Framework: ${options.framework}`);
300
- console.log(`Distributed: ${options.distributed ? 'yes' : 'no'}`);
301
- console.log(`Agents: ${options.agents ? 'yes' : 'no'}`);
302
- console.log(`WebLLM: ${options.webllm ? 'yes — natural-language app builder' : 'no'}`);
303
- console.log(`Capabilities: ${options.capabilities}`);
306
+ if (options.framework === 'react' && !options.agents) {
307
+ console.log('\nNote: the React starter always includes its assistants; --no-agents applies to the vanilla template.');
308
+ }
309
+ options.agents = agentsIncluded(options);
310
+ console.log('');
311
+ for (const line of describeSetup(options))
312
+ console.log(line);
304
313
  console.log('\n═══════════════════════════════════════════════\n');
305
314
  const projectDir = path.resolve(process.cwd(), projectName);
306
315
  let projectCreated = false;
@@ -196,12 +196,25 @@ ${hasAgents ? ` agent WorkspaceAssistant {
196
196
  tsconfig.compilerOptions.types = ['vite/client'];
197
197
  }
198
198
  fs.writeFileSync(path.join(projectDir, 'tsconfig.json'), JSON.stringify(tsconfig, null, 2));
199
- // Browsers do not expose VITE_* through process.env, so a local database must be handed its
199
+ // Browsers do not expose VITE_* through process.env, so the database must be handed its
200
200
  // namespace explicitly; otherwise it opens the shared `default` store and `feltdb dev` cannot
201
201
  // recognise the application. A managed (remote) runtime resolves its target from its URL.
202
+ //
203
+ // The generated application runs in the browser whatever the deployment target. For a
204
+ // browser target its state is the browser's own (IndexedDB). For a Node.js or self-hosted
205
+ // target the state lives in a FeltDB authority, so the browser application connects to it:
206
+ // the URL the developer configured, else the authority `feltdb dev` started. Opening a
207
+ // browser store there instead would leave the application writing data Studio and the
208
+ // authority never see.
209
+ const namespaceExpression = `import.meta.env.VITE_FELTDB_NAMESPACE || ${JSON.stringify(applicationName)}`;
202
210
  const createDatabaseCall = runtime === 'managed'
203
211
  ? 'createFeltDB()'
204
- : `createFeltDB({ namespace: import.meta.env.VITE_FELTDB_NAMESPACE || ${JSON.stringify(applicationName)} })`;
212
+ : runtime === 'browser'
213
+ ? `createFeltDB({ namespace: ${namespaceExpression} })`
214
+ : `createFeltDB({
215
+ namespace: ${namespaceExpression},
216
+ server: { url: import.meta.env.VITE_FELTDB_URL || import.meta.env.VITE_FELTDB_AUTHORITY_URL || 'http://127.0.0.1:7700' },
217
+ })`;
205
218
  // Create main application files
206
219
  const feltdbTs = `import { configureDevelopmentRuntimeBridge, createFeltDB } from '@feltdb/core';
207
220
 
@@ -1222,6 +1235,8 @@ main().catch(console.error);
1222
1235
  <head>
1223
1236
  <meta charset="UTF-8">
1224
1237
  <meta name="viewport" content="width=device-width, initial-scale=1.0">
1238
+ <!-- An inline icon: without one the browser requests /favicon.ico and logs a 404. -->
1239
+ <link rel="icon" href="data:,">
1225
1240
  <title>${applicationName}</title>
1226
1241
  </head>
1227
1242
  <body>
@@ -1,4 +1,4 @@
1
1
  // One release train keeps generated applications installable. The repository
2
2
  // validation script checks these values against every workspace manifest.
3
- export const FELTDB_PACKAGE_VERSION = '0.12.2';
3
+ export const FELTDB_PACKAGE_VERSION = '0.13.0';
4
4
  export const feltdbPackageRange = `^${FELTDB_PACKAGE_VERSION}`;
@@ -494,7 +494,7 @@ checksum = "da7c62ceae207dd37ea5b845da6a0696c799f85e97da1ab5b7910be3c1c80223"
494
494
 
495
495
  [[package]]
496
496
  name = "feltdb"
497
- version = "0.2.1"
497
+ version = "0.2.2"
498
498
  dependencies = [
499
499
  "async-stream",
500
500
  "async-trait",
@@ -513,7 +513,7 @@ dependencies = [
513
513
 
514
514
  [[package]]
515
515
  name = "feltdb-client"
516
- version = "0.2.1"
516
+ version = "0.2.2"
517
517
  dependencies = [
518
518
  "feltdb",
519
519
  "reqwest",
@@ -526,7 +526,7 @@ dependencies = [
526
526
 
527
527
  [[package]]
528
528
  name = "feltdb-server"
529
- version = "0.2.1"
529
+ version = "0.2.2"
530
530
  dependencies = [
531
531
  "argon2",
532
532
  "async-stream",
@@ -552,7 +552,7 @@ dependencies = [
552
552
 
553
553
  [[package]]
554
554
  name = "feltdb-wasm"
555
- version = "0.2.1"
555
+ version = "0.2.2"
556
556
  dependencies = [
557
557
  "feltdb",
558
558
  "getrandom 0.2.17",
@@ -14,5 +14,5 @@ exclude = [
14
14
  resolver = "2"
15
15
 
16
16
  [workspace.package]
17
- version = "0.2.1"
17
+ version = "0.2.2"
18
18
  edition = "2021"
@@ -486,6 +486,48 @@ impl WorkerMeshStore {
486
486
  }
487
487
  self.persist()
488
488
  }
489
+ /// Recover work whose claim lease expired, and release those claims from
490
+ /// the workers that held them so their capacity is not held by work they
491
+ /// no longer own. Recovery itself — `WorkerLost`, the attempt budget, the
492
+ /// decision — is `WorkloadStore::recover_expired`. Worker liveness is not
493
+ /// judged here; that is `detect_failures`.
494
+ pub fn recover_expired_work(
495
+ &mut self,
496
+ now: i64,
497
+ workloads: &mut WorkloadStore,
498
+ ) -> Result<Vec<String>, WorkerMeshError> {
499
+ let expired: Vec<(String, String, String)> = workloads
500
+ .workloads
501
+ .values()
502
+ .filter_map(|w| w.current_claim.as_ref().map(|claim| (w, claim)))
503
+ .filter(|(_, claim)| claim.expires_at <= now)
504
+ .map(|(w, claim)| {
505
+ (
506
+ w.workload_id.clone(),
507
+ claim.worker_id.clone(),
508
+ claim.claim_id.clone(),
509
+ )
510
+ })
511
+ .collect();
512
+ let recovered = workloads.recover_expired(now)?;
513
+ for (workload_id, worker_id, claim_id) in expired {
514
+ // Release a claim only once the committed workload no longer holds
515
+ // it, whichever writer of the store recovered it.
516
+ let still_current = workloads
517
+ .workloads
518
+ .get(&workload_id)
519
+ .and_then(|w| w.current_claim.as_ref())
520
+ .is_some_and(|claim| claim.claim_id == claim_id);
521
+ let held = self
522
+ .workers
523
+ .get(&worker_id)
524
+ .is_some_and(|worker| worker.claims.contains(&claim_id));
525
+ if held && !still_current {
526
+ self.release_claim(&worker_id, &claim_id)?;
527
+ }
528
+ }
529
+ Ok(recovered)
530
+ }
489
531
  pub fn detect_failures(
490
532
  &mut self,
491
533
  now: i64,
@@ -507,7 +549,7 @@ impl WorkerMeshStore {
507
549
  worker.claims.clear();
508
550
  }
509
551
  }
510
- workloads.recover_expired(now)?;
552
+ self.recover_expired_work(now, workloads)?;
511
553
  self.persist()?;
512
554
  Ok(failed)
513
555
  }
@@ -852,6 +894,167 @@ mod tests {
852
894
  let _ = fs::remove_file(w);
853
895
  }
854
896
  #[test]
897
+ fn a_claim_recovered_by_another_writer_is_still_released() {
898
+ let (m, wpath, mut mesh, mut workloads, grant) = setup("release-raced");
899
+ let worker = register(&mut mesh, grant.clone());
900
+ let heartbeat = |timestamp| WorkerHeartbeat {
901
+ worker_id: worker.worker_id.clone(),
902
+ version: "1".into(),
903
+ capability_version: 1,
904
+ capacity: WorkerCapacity {
905
+ concurrency: 1,
906
+ active: 0,
907
+ },
908
+ health: WorkerHealth::Healthy,
909
+ timestamp,
910
+ running_claims: BTreeSet::new(),
911
+ };
912
+ mesh.heartbeat(&worker.worker_id, heartbeat(11), 2).unwrap();
913
+ let subject = grant.grant.subject.clone();
914
+ let create = |key: &str| CreateWorkload {
915
+ tenant_id: "t".into(),
916
+ application_id: "a".into(),
917
+ environment: "production".into(),
918
+ definition_id: "extract".into(),
919
+ definition_revision: "r".into(),
920
+ trigger: "manual".into(),
921
+ input: serde_json::json!({"file": key}),
922
+ requested_by: subject.clone(),
923
+ session_id: None,
924
+ authorization_snapshot: grant.clone(),
925
+ capability_snapshot: ["documents.extract".into()].into(),
926
+ connection_scopes: BTreeSet::new(),
927
+ priority: 0,
928
+ idempotency_key: key.into(),
929
+ retry_policy: WorkloadRetryPolicy::default(),
930
+ timeouts: WorkloadTimeouts::default(),
931
+ dependencies: vec![],
932
+ not_before: None,
933
+ created_at: 11,
934
+ };
935
+ let first = workloads.create(create("first")).unwrap();
936
+ let second = workloads.create(create("second")).unwrap();
937
+ mesh.place_and_claim(
938
+ &mut workloads,
939
+ &first.workload_id,
940
+ &worker.worker_id,
941
+ 12,
942
+ 1000,
943
+ )
944
+ .unwrap();
945
+ assert_eq!(mesh.workers[&worker.worker_id].capacity.active, 1);
946
+ // Another writer of the same store recovers the work first.
947
+ let mut other = WorkloadStore::from_db(workloads.db_for_tests()).unwrap();
948
+ assert_eq!(
949
+ other.recover_expired(14).unwrap(),
950
+ vec![first.workload_id.clone()]
951
+ );
952
+ // This node's pass loses the guarded commit, adopts the committed
953
+ // workload, does not report it as its own recovery, and still frees the
954
+ // worker, whose claim is no longer the workload's.
955
+ assert!(mesh
956
+ .recover_expired_work(14, &mut workloads)
957
+ .unwrap()
958
+ .is_empty());
959
+ let recovered = workloads.get(&first.workload_id).unwrap();
960
+ assert_eq!(recovered.state, crate::workload::WorkloadState::Ready);
961
+ assert_eq!(
962
+ recovered
963
+ .execution_history
964
+ .iter()
965
+ .filter(|event| event.to == crate::workload::WorkloadState::Recovering)
966
+ .count(),
967
+ 1
968
+ );
969
+ assert!(mesh.workers[&worker.worker_id].claims.is_empty());
970
+ assert_eq!(mesh.workers[&worker.worker_id].capacity.active, 0);
971
+ let _ = second;
972
+ let _ = fs::remove_file(m);
973
+ let _ = fs::remove_file(wpath);
974
+ }
975
+ #[test]
976
+ fn recovering_an_expired_claim_releases_it_from_a_live_worker() {
977
+ let (m, wpath, mut mesh, mut workloads, grant) = setup("release-expired");
978
+ let worker = register(&mut mesh, grant.clone());
979
+ let heartbeat = |timestamp| WorkerHeartbeat {
980
+ worker_id: worker.worker_id.clone(),
981
+ version: "1".into(),
982
+ capability_version: 1,
983
+ capacity: WorkerCapacity {
984
+ concurrency: 1,
985
+ active: 0,
986
+ },
987
+ health: WorkerHealth::Healthy,
988
+ timestamp,
989
+ running_claims: BTreeSet::new(),
990
+ };
991
+ mesh.heartbeat(&worker.worker_id, heartbeat(11), 2).unwrap();
992
+ let subject = grant.grant.subject.clone();
993
+ let create = |key: &str| CreateWorkload {
994
+ tenant_id: "t".into(),
995
+ application_id: "a".into(),
996
+ environment: "production".into(),
997
+ definition_id: "extract".into(),
998
+ definition_revision: "r".into(),
999
+ trigger: "manual".into(),
1000
+ input: serde_json::json!({"file": key}),
1001
+ requested_by: subject.clone(),
1002
+ session_id: None,
1003
+ authorization_snapshot: grant.clone(),
1004
+ capability_snapshot: ["documents.extract".into()].into(),
1005
+ connection_scopes: BTreeSet::new(),
1006
+ priority: 0,
1007
+ idempotency_key: key.into(),
1008
+ retry_policy: WorkloadRetryPolicy::default(),
1009
+ timeouts: WorkloadTimeouts::default(),
1010
+ dependencies: vec![],
1011
+ not_before: None,
1012
+ created_at: 11,
1013
+ };
1014
+ let first = workloads.create(create("first")).unwrap();
1015
+ let second = workloads.create(create("second")).unwrap();
1016
+ mesh.place_and_claim(
1017
+ &mut workloads,
1018
+ &first.workload_id,
1019
+ &worker.worker_id,
1020
+ 12,
1021
+ 1000,
1022
+ )
1023
+ .unwrap();
1024
+ assert_eq!(mesh.workers[&worker.worker_id].capacity.active, 1);
1025
+ // The lease expired at 13. The server's recovery is claim recovery: it
1026
+ // recovers the work and frees the worker, and does not judge liveness.
1027
+ let recovered = mesh.recover_expired_work(14, &mut workloads).unwrap();
1028
+ assert_eq!(recovered, vec![first.workload_id.clone()]);
1029
+ let live = &mesh.workers[&worker.worker_id];
1030
+ assert!(live.claims.is_empty());
1031
+ assert_eq!(live.capacity.active, 0);
1032
+ assert_eq!(live.lifecycle, WorkerLifecycle::Ready);
1033
+ assert_eq!(
1034
+ workloads.get(&first.workload_id).unwrap().state,
1035
+ crate::workload::WorkloadState::Ready
1036
+ );
1037
+ // Its capacity is usable again.
1038
+ mesh.place_and_claim(
1039
+ &mut workloads,
1040
+ &second.workload_id,
1041
+ &worker.worker_id,
1042
+ 15,
1043
+ 10_000,
1044
+ )
1045
+ .unwrap();
1046
+ // Long past its heartbeat timeout, claim recovery still leaves the
1047
+ // worker's lifecycle alone; only `detect_failures` marks it offline.
1048
+ mesh.recover_expired_work(1_000, &mut workloads).unwrap();
1049
+ assert_ne!(
1050
+ mesh.workers[&worker.worker_id].lifecycle,
1051
+ WorkerLifecycle::Offline
1052
+ );
1053
+ assert!(mesh.workers[&worker.worker_id].claims.is_empty());
1054
+ let _ = fs::remove_file(m);
1055
+ let _ = fs::remove_file(wpath);
1056
+ }
1057
+ #[test]
855
1058
  fn placement_claim_fencing_failure_and_recovery() {
856
1059
  let (m, wpath, mut mesh, mut workloads, grant) = setup("claim");
857
1060
  let worker = register(&mut mesh, grant.clone());