@opendatalabs/personal-server-ts-server 0.0.1-canary.016adf9 → 0.0.1-canary.052d47e

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (213) hide show
  1. package/dist/api-auth.d.ts +31 -0
  2. package/dist/api-auth.d.ts.map +1 -0
  3. package/dist/api-auth.js +148 -0
  4. package/dist/api-auth.js.map +1 -0
  5. package/dist/api.d.ts +4 -0
  6. package/dist/api.d.ts.map +1 -1
  7. package/dist/api.js +3 -0
  8. package/dist/api.js.map +1 -1
  9. package/dist/app.d.ts +77 -2
  10. package/dist/app.d.ts.map +1 -1
  11. package/dist/app.js +165 -9
  12. package/dist/app.js.map +1 -1
  13. package/dist/bootstrap.d.ts +36 -3
  14. package/dist/bootstrap.d.ts.map +1 -1
  15. package/dist/bootstrap.js +434 -39
  16. package/dist/bootstrap.js.map +1 -1
  17. package/dist/client.d.ts +17 -0
  18. package/dist/client.d.ts.map +1 -0
  19. package/dist/client.js +369 -0
  20. package/dist/client.js.map +1 -0
  21. package/dist/config/defaults.d.ts +12 -0
  22. package/dist/config/defaults.d.ts.map +1 -0
  23. package/dist/config/defaults.js +14 -0
  24. package/dist/config/defaults.js.map +1 -0
  25. package/dist/config/index.d.ts +4 -0
  26. package/dist/config/index.d.ts.map +1 -0
  27. package/dist/config/index.js +4 -0
  28. package/dist/config/index.js.map +1 -0
  29. package/dist/config/loader.d.ts +8 -0
  30. package/dist/config/loader.d.ts.map +1 -0
  31. package/dist/config/loader.js +78 -0
  32. package/dist/config/loader.js.map +1 -0
  33. package/dist/config/paths.d.ts +9 -0
  34. package/dist/config/paths.d.ts.map +1 -0
  35. package/dist/config/paths.js +22 -0
  36. package/dist/config/paths.js.map +1 -0
  37. package/dist/enclave-main.d.ts +10 -0
  38. package/dist/enclave-main.d.ts.map +1 -0
  39. package/dist/enclave-main.js +115 -0
  40. package/dist/enclave-main.js.map +1 -0
  41. package/dist/hono-context.d.ts +14 -0
  42. package/dist/hono-context.d.ts.map +1 -0
  43. package/dist/hono-context.js +2 -0
  44. package/dist/hono-context.js.map +1 -0
  45. package/dist/index.js +48 -12
  46. package/dist/index.js.map +1 -1
  47. package/dist/jobs/types.d.ts +17 -0
  48. package/dist/jobs/types.d.ts.map +1 -0
  49. package/dist/jobs/types.js +3 -0
  50. package/dist/jobs/types.js.map +1 -0
  51. package/dist/jobs/worker.d.ts +30 -0
  52. package/dist/jobs/worker.d.ts.map +1 -0
  53. package/dist/jobs/worker.js +217 -0
  54. package/dist/jobs/worker.js.map +1 -0
  55. package/dist/keys/server-account.d.ts +28 -0
  56. package/dist/keys/server-account.d.ts.map +1 -0
  57. package/dist/keys/server-account.js +48 -0
  58. package/dist/keys/server-account.js.map +1 -0
  59. package/dist/listen.d.ts +11 -0
  60. package/dist/listen.d.ts.map +1 -0
  61. package/dist/listen.js +23 -0
  62. package/dist/listen.js.map +1 -0
  63. package/dist/logger/index.d.ts +5 -0
  64. package/dist/logger/index.d.ts.map +1 -0
  65. package/dist/logger/index.js +9 -0
  66. package/dist/logger/index.js.map +1 -0
  67. package/dist/logging/access-log.d.ts +4 -0
  68. package/dist/logging/access-log.d.ts.map +1 -0
  69. package/dist/logging/access-log.js +25 -0
  70. package/dist/logging/access-log.js.map +1 -0
  71. package/dist/logging/access-reader.d.ts +19 -0
  72. package/dist/logging/access-reader.d.ts.map +1 -0
  73. package/dist/logging/access-reader.js +50 -0
  74. package/dist/logging/access-reader.js.map +1 -0
  75. package/dist/middleware/access-log.d.ts.map +1 -1
  76. package/dist/middleware/access-log.js +8 -2
  77. package/dist/middleware/access-log.js.map +1 -1
  78. package/dist/middleware/builder-check.d.ts +2 -2
  79. package/dist/middleware/builder-check.d.ts.map +1 -1
  80. package/dist/middleware/builder-check.js +7 -2
  81. package/dist/middleware/builder-check.js.map +1 -1
  82. package/dist/middleware/cors.d.ts +18 -0
  83. package/dist/middleware/cors.d.ts.map +1 -0
  84. package/dist/middleware/cors.js +51 -0
  85. package/dist/middleware/cors.js.map +1 -0
  86. package/dist/middleware/owner-check.js +1 -1
  87. package/dist/middleware/owner-check.js.map +1 -1
  88. package/dist/middleware/web3-auth.d.ts +12 -0
  89. package/dist/middleware/web3-auth.d.ts.map +1 -1
  90. package/dist/middleware/web3-auth.js +27 -31
  91. package/dist/middleware/web3-auth.js.map +1 -1
  92. package/dist/migrations/detect-stuck-version-ledger.d.ts +43 -0
  93. package/dist/migrations/detect-stuck-version-ledger.d.ts.map +1 -0
  94. package/dist/migrations/detect-stuck-version-ledger.js +73 -0
  95. package/dist/migrations/detect-stuck-version-ledger.js.map +1 -0
  96. package/dist/migrations/local-state.d.ts +35 -0
  97. package/dist/migrations/local-state.d.ts.map +1 -0
  98. package/dist/migrations/local-state.js +133 -0
  99. package/dist/migrations/local-state.js.map +1 -0
  100. package/dist/migrations/state-migrations.d.ts +78 -0
  101. package/dist/migrations/state-migrations.d.ts.map +1 -0
  102. package/dist/migrations/state-migrations.js +72 -0
  103. package/dist/migrations/state-migrations.js.map +1 -0
  104. package/dist/pending-blob-deletions.d.ts +10 -0
  105. package/dist/pending-blob-deletions.d.ts.map +1 -0
  106. package/dist/pending-blob-deletions.js +38 -0
  107. package/dist/pending-blob-deletions.js.map +1 -0
  108. package/dist/routes/access-logs.d.ts +7 -0
  109. package/dist/routes/access-logs.d.ts.map +1 -1
  110. package/dist/routes/access-logs.js +11 -17
  111. package/dist/routes/access-logs.js.map +1 -1
  112. package/dist/routes/auth-device.d.ts +49 -0
  113. package/dist/routes/auth-device.d.ts.map +1 -0
  114. package/dist/routes/auth-device.js +581 -0
  115. package/dist/routes/auth-device.js.map +1 -0
  116. package/dist/routes/data.d.ts +67 -2
  117. package/dist/routes/data.d.ts.map +1 -1
  118. package/dist/routes/data.js +41 -201
  119. package/dist/routes/data.js.map +1 -1
  120. package/dist/routes/derivatives.d.ts +33 -0
  121. package/dist/routes/derivatives.d.ts.map +1 -0
  122. package/dist/routes/derivatives.js +33 -0
  123. package/dist/routes/derivatives.js.map +1 -0
  124. package/dist/routes/enclave-jobs.d.ts +9 -0
  125. package/dist/routes/enclave-jobs.d.ts.map +1 -0
  126. package/dist/routes/enclave-jobs.js +104 -0
  127. package/dist/routes/enclave-jobs.js.map +1 -0
  128. package/dist/routes/grants.d.ts +9 -2
  129. package/dist/routes/grants.d.ts.map +1 -1
  130. package/dist/routes/grants.js +18 -196
  131. package/dist/routes/grants.js.map +1 -1
  132. package/dist/routes/health.d.ts +7 -1
  133. package/dist/routes/health.d.ts.map +1 -1
  134. package/dist/routes/health.js +39 -2
  135. package/dist/routes/health.js.map +1 -1
  136. package/dist/routes/mcp.d.ts +115 -0
  137. package/dist/routes/mcp.d.ts.map +1 -0
  138. package/dist/routes/mcp.js +676 -0
  139. package/dist/routes/mcp.js.map +1 -0
  140. package/dist/routes/oauth-token.d.ts +56 -0
  141. package/dist/routes/oauth-token.d.ts.map +1 -0
  142. package/dist/routes/oauth-token.js +49 -0
  143. package/dist/routes/oauth-token.js.map +1 -0
  144. package/dist/routes/sync.d.ts +7 -0
  145. package/dist/routes/sync.d.ts.map +1 -1
  146. package/dist/routes/sync.js +12 -39
  147. package/dist/routes/sync.js.map +1 -1
  148. package/dist/routes/ui-config.d.ts.map +1 -1
  149. package/dist/routes/ui-config.js +17 -82
  150. package/dist/routes/ui-config.js.map +1 -1
  151. package/dist/routes/ui-registration.d.ts +9 -0
  152. package/dist/routes/ui-registration.d.ts.map +1 -0
  153. package/dist/routes/ui-registration.js +217 -0
  154. package/dist/routes/ui-registration.js.map +1 -0
  155. package/dist/routes/ui.d.ts +1 -0
  156. package/dist/routes/ui.d.ts.map +1 -1
  157. package/dist/routes/ui.js +46 -4
  158. package/dist/routes/ui.js.map +1 -1
  159. package/dist/routes/write-session.d.ts +37 -0
  160. package/dist/routes/write-session.d.ts.map +1 -0
  161. package/dist/routes/write-session.js +34 -0
  162. package/dist/routes/write-session.js.map +1 -0
  163. package/dist/storage/hierarchy.d.ts +36 -0
  164. package/dist/storage/hierarchy.d.ts.map +1 -0
  165. package/dist/storage/hierarchy.js +282 -0
  166. package/dist/storage/hierarchy.js.map +1 -0
  167. package/dist/storage/index-manager.d.ts +60 -0
  168. package/dist/storage/index-manager.d.ts.map +1 -0
  169. package/dist/storage/index-manager.js +174 -0
  170. package/dist/storage/index-manager.js.map +1 -0
  171. package/dist/storage/index-schema.d.ts +5 -0
  172. package/dist/storage/index-schema.d.ts.map +1 -0
  173. package/dist/storage/index-schema.js +69 -0
  174. package/dist/storage/index-schema.js.map +1 -0
  175. package/dist/storage/node-data-storage.d.ts +9 -0
  176. package/dist/storage/node-data-storage.d.ts.map +1 -0
  177. package/dist/storage/node-data-storage.js +105 -0
  178. package/dist/storage/node-data-storage.js.map +1 -0
  179. package/dist/storage/question-store.d.ts +9 -0
  180. package/dist/storage/question-store.d.ts.map +1 -0
  181. package/dist/storage/question-store.js +151 -0
  182. package/dist/storage/question-store.js.map +1 -0
  183. package/dist/sync-cursor.d.ts +15 -0
  184. package/dist/sync-cursor.d.ts.map +1 -0
  185. package/dist/sync-cursor.js +110 -0
  186. package/dist/sync-cursor.js.map +1 -0
  187. package/dist/token-store.d.ts +22 -0
  188. package/dist/token-store.d.ts.map +1 -0
  189. package/dist/token-store.js +120 -0
  190. package/dist/token-store.js.map +1 -0
  191. package/dist/tunnel/auth.d.ts +4 -0
  192. package/dist/tunnel/auth.d.ts.map +1 -1
  193. package/dist/tunnel/auth.js +2 -1
  194. package/dist/tunnel/auth.js.map +1 -1
  195. package/dist/tunnel/config.d.ts +1 -0
  196. package/dist/tunnel/config.d.ts.map +1 -1
  197. package/dist/tunnel/config.js +1 -1
  198. package/dist/tunnel/config.js.map +1 -1
  199. package/dist/tunnel/manager.d.ts +21 -1
  200. package/dist/tunnel/manager.d.ts.map +1 -1
  201. package/dist/tunnel/manager.js +78 -23
  202. package/dist/tunnel/manager.js.map +1 -1
  203. package/dist/ui/browser_tls_rustls_bg.wasm +0 -0
  204. package/dist/ui/index.html +2192 -0
  205. package/dist/ui/ps-lite-debug.d.ts +93 -0
  206. package/dist/ui/ps-lite-debug.d.ts.map +1 -0
  207. package/dist/ui/ps-lite-debug.js +135347 -0
  208. package/dist/ui/ps-lite-debug.js.map +1 -0
  209. package/package.json +20 -6
  210. package/dist/middleware/grant-check.d.ts +0 -12
  211. package/dist/middleware/grant-check.d.ts.map +0 -1
  212. package/dist/middleware/grant-check.js +0 -90
  213. package/dist/middleware/grant-check.js.map +0 -1
package/dist/bootstrap.js CHANGED
@@ -2,48 +2,118 @@ import { mkdir } from "node:fs/promises";
2
2
  import { createRequire } from "node:module";
3
3
  import { randomUUID } from "node:crypto";
4
4
  import { join } from "node:path";
5
+ import { privateKeyToAccount } from "viem/accounts";
5
6
  const require = createRequire(import.meta.url);
6
7
  const pkg = require("../package.json");
7
- import { DEFAULT_ROOT_PATH, resolveRootPath, } from "@opendatalabs/personal-server-ts-core/config";
8
- import { createLogger, } from "@opendatalabs/personal-server-ts-core/logger";
9
- import { initializeDatabase, createIndexManager, } from "@opendatalabs/personal-server-ts-core/storage/index";
10
- import { createGatewayClient } from "@opendatalabs/personal-server-ts-core/gateway";
11
- import { createAccessLogWriter } from "@opendatalabs/personal-server-ts-core/logging/access-log";
12
- import { createAccessLogReader } from "@opendatalabs/personal-server-ts-core/logging/access-reader";
13
- import { deriveMasterKey, recoverServerOwner, loadOrCreateServerAccount, } from "@opendatalabs/personal-server-ts-core/keys";
14
- import { createServerSigner, createRequestSigner, } from "@opendatalabs/personal-server-ts-core/signing";
15
- import { createSyncCursor, createSyncManager, } from "@opendatalabs/personal-server-ts-core/sync";
16
- import { createVanaStorageAdapter } from "@opendatalabs/personal-server-ts-core/storage/adapters";
8
+ import { DEFAULT_ROOT_PATH, resolveRootPath } from "./config/index.js";
9
+ import { createLogger } from "./logger/index.js";
10
+ import { initializeDatabase } from "./storage/index-schema.js";
11
+ import { createIndexManager, } from "./storage/index-manager.js";
12
+ import { createGatewayClient, NodeECIESProvider, } from "@opendatalabs/vana-sdk/node";
13
+ import { createAccessLogWriter } from "./logging/access-log.js";
14
+ import { createAccessLogReader } from "./logging/access-reader.js";
15
+ import { deriveMasterKey, recoverServerOwner, } from "@opendatalabs/vana-sdk/node";
16
+ import { loadOrCreateServerAccount } from "./keys/server-account.js";
17
+ import { createRequestSigner, createServerSigner, } from "@opendatalabs/personal-server-ts-core/signing";
18
+ import { createGatewayLineageClient, } from "@opendatalabs/personal-server-ts-core/lineage";
19
+ import { createSyncCursor } from "./sync-cursor.js";
20
+ import { createGatewayDataPointFeed, createGatewayDeleteDataPort, createScopeDeletionTracker, createSyncManager, } from "@opendatalabs/personal-server-ts-core/sync";
21
+ import { createVanaSyncStorageAdapter, resolveVanaStorageEndpoint, } from "@opendatalabs/personal-server-ts-core/storage/adapters";
22
+ import { createFilePendingBlobDeletionStore } from "./pending-blob-deletions.js";
17
23
  import { createApp } from "./app.js";
18
24
  import { generateDevToken } from "./dev-token.js";
25
+ import { migrateLocalState } from "./migrations/local-state.js";
26
+ import { createTokenStore } from "./token-store.js";
19
27
  import { TunnelManager, ensureFrpcBinary } from "./tunnel/index.js";
28
+ import { createNodeDataStorage } from "./storage/node-data-storage.js";
29
+ import { createSqliteQuestionStore } from "./storage/question-store.js";
30
+ import { computeQuestion, createOpenAiCompatibleInferenceProvider, createPhalaE2eeEncryption, createRecomputeScheduler, } from "@opendatalabs/personal-server-ts-core/derivatives";
31
+ import { executeJob } from "./jobs/worker.js";
32
+ const TRAILING_SLASHES = /\/+$/;
33
+ const DEFAULT_LOCAL_APPROVAL_PORT = 34127;
34
+ function resolveLocalApprovalPort(serverPort) {
35
+ const raw = process.env.LOCAL_AUTH_PORT;
36
+ if (raw !== undefined) {
37
+ const parsed = Number(raw);
38
+ if (Number.isInteger(parsed) &&
39
+ parsed >= 1 &&
40
+ parsed <= 65535 &&
41
+ parsed !== serverPort) {
42
+ return parsed;
43
+ }
44
+ }
45
+ const adjacentPort = serverPort + 1;
46
+ if (adjacentPort <= 65535) {
47
+ return adjacentPort;
48
+ }
49
+ return DEFAULT_LOCAL_APPROVAL_PORT;
50
+ }
20
51
  export async function createServer(config, options) {
21
52
  const logger = createLogger(config.logging);
22
53
  const startedAt = new Date();
54
+ const isEnclave = options?.profile === "enclave";
55
+ if (isEnclave && process.env.VANA_OWNER_PRIVATE_KEY) {
56
+ throw new Error("VANA_OWNER_PRIVATE_KEY is forbidden in enclave profile");
57
+ }
23
58
  const storageRoot = resolveRootPath(options?.rootPath ?? options?.serverDir ?? DEFAULT_ROOT_PATH);
24
59
  const dataDir = options?.dataDir ?? join(storageRoot, "data");
25
60
  const indexPath = join(storageRoot, "index.db");
26
61
  const configPath = join(storageRoot, "config.json");
62
+ const syncCursorPath = join(storageRoot, "sync-cursor.json");
63
+ const pendingBlobDeletionsPath = join(storageRoot, "pending-blob-deletions.json");
64
+ const tokensPath = join(storageRoot, "tokens.json");
27
65
  await mkdir(storageRoot, { recursive: true });
28
66
  await mkdir(dataDir, { recursive: true });
29
67
  const db = initializeDatabase(indexPath);
68
+ await migrateLocalState({
69
+ storageRoot,
70
+ dataDir,
71
+ configPath,
72
+ syncCursorPath,
73
+ tokensPath,
74
+ db,
75
+ logger,
76
+ });
30
77
  const indexManager = createIndexManager(db);
31
78
  const hierarchyOptions = { dataDir };
32
- const gatewayClient = createGatewayClient(config.gateway.url);
79
+ const dataStorage = createNodeDataStorage({ indexManager, hierarchyOptions });
80
+ const gatewayClient = options?.gatewayClient ?? createGatewayClient(config.gateway.url);
81
+ // X402 payment enforcement on data reads. When config.payment.enabled is
82
+ // true, GET /v1/data/:scope requires builders to supply a signed X-PAYMENT
83
+ // header on every read; the server forwards to gateway.payForOperation.
84
+ // See packages/core/src/payment/x402.ts. Off-by-default for development
85
+ // and test setups where signing payment per read is friction.
86
+ if (config.payment.enabled) {
87
+ logger.info("X402 payment enforcement enabled on data reads");
88
+ }
33
89
  // Derive server owner from VANA_MASTER_KEY_SIGNATURE env var
34
- const masterKeySignature = process.env.VANA_MASTER_KEY_SIGNATURE;
90
+ const masterKeySignature = options?.ownerSignature ??
91
+ process.env.VANA_MASTER_KEY_SIGNATURE;
92
+ const ownerPrivateKey = process.env.VANA_OWNER_PRIVATE_KEY;
93
+ const ownerTunnelSigner = ownerPrivateKey
94
+ ? privateKeyToAccount(ownerPrivateKey.startsWith("0x")
95
+ ? ownerPrivateKey
96
+ : `0x${ownerPrivateKey}`)
97
+ : null;
35
98
  let serverOwner;
36
99
  let serverAccount;
37
100
  let serverSigner;
38
101
  let identity;
39
102
  if (masterKeySignature) {
40
- serverOwner = await recoverServerOwner(masterKeySignature);
103
+ const recoveredServerOwner = await recoverServerOwner(masterKeySignature);
104
+ serverOwner = recoveredServerOwner;
41
105
  deriveMasterKey(masterKeySignature); // validate signature format
42
106
  logger.info({ owner: serverOwner }, "Server owner derived from master key");
43
- // Load or create server keypair from disk
107
+ if (ownerTunnelSigner &&
108
+ ownerTunnelSigner.address.toLowerCase() !==
109
+ recoveredServerOwner.toLowerCase()) {
110
+ throw new Error("VANA_OWNER_PRIVATE_KEY does not match VANA_MASTER_KEY_SIGNATURE owner");
111
+ }
112
+ // Enclaves receive public identity only; standard mode owns its key file.
44
113
  const keyPath = join(storageRoot, "key.json");
45
- serverAccount = loadOrCreateServerAccount(keyPath);
46
- logger.info({ owner: serverOwner, serverAddress: serverAccount.address }, "Server signing account loaded");
114
+ serverAccount =
115
+ options?.serverAccount ?? loadOrCreateServerAccount(keyPath);
116
+ logger.info({ owner: serverOwner, serverAddress: serverAccount.address }, "Server account loaded");
47
117
  serverSigner = createServerSigner(serverAccount, {
48
118
  chainId: config.gateway.chainId,
49
119
  contracts: config.gateway.contracts,
@@ -58,9 +128,33 @@ export async function createServer(config, options) {
58
128
  else {
59
129
  logger.warn("VANA_MASTER_KEY_SIGNATURE not set — owner-restricted endpoints will return 500");
60
130
  }
131
+ // Resolve PS_ACCESS_TOKEN for cloud control-plane / bootstrap auth.
132
+ // This is not the per-login CLI session token returned to users.
133
+ let accessToken = process.env.PS_ACCESS_TOKEN;
134
+ if (!accessToken && process.env.CLOUD_MODE === "true") {
135
+ try {
136
+ const metadataRes = await fetch("http://metadata.google.internal/computeMetadata/v1/instance/attributes/ps-access-token", {
137
+ headers: { "Metadata-Flavor": "Google" },
138
+ signal: AbortSignal.timeout(2000),
139
+ });
140
+ if (metadataRes.ok) {
141
+ const token = (await metadataRes.text()).trim();
142
+ if (token.length > 0) {
143
+ accessToken = token;
144
+ logger.info("PS control-plane token loaded from instance metadata");
145
+ }
146
+ }
147
+ }
148
+ catch {
149
+ logger.debug("Could not read PS control-plane token from instance metadata");
150
+ }
151
+ }
152
+ if (accessToken) {
153
+ logger.info("PS_ACCESS_TOKEN configured — control-plane bearer auth enabled");
154
+ }
61
155
  // Download frpc binary eagerly (auth-independent) so it's ready when the user signs in
62
156
  let frpcBinaryPath = "";
63
- if (config.tunnel.enabled) {
157
+ if (config.tunnel.enabled && !isEnclave) {
64
158
  try {
65
159
  frpcBinaryPath = await ensureFrpcBinary(storageRoot, {
66
160
  log: (msg) => logger.info(msg),
@@ -70,45 +164,181 @@ export async function createServer(config, options) {
70
164
  logger.warn({ err }, "Failed to download frpc binary - tunnel disabled");
71
165
  }
72
166
  }
167
+ // Derivative data: the gateway lineage client signs lineage reads with the
168
+ // server key (the gateway treats a registered server as the owner's) and
169
+ // registers derivatives with their lineage. Needs the server account.
170
+ // The one server-key request signer: lineage reads, and the inference
171
+ // relay calls below, authenticate to the gateway with the same scheme.
172
+ const requestSigner = serverAccount && !isEnclave
173
+ ? createRequestSigner(serverAccount)
174
+ : undefined;
175
+ const lineageGateway = requestSigner
176
+ ? createGatewayLineageClient({
177
+ gatewayUrl: config.gateway.url,
178
+ requestSigner,
179
+ })
180
+ : undefined;
73
181
  // --- Sync engine setup ---
74
182
  let syncManager = null;
183
+ // Deletion-aware registry view. Independent of sync: the data route uses
184
+ // it to answer 410 for scopes the owner deleted even when sync is off.
185
+ const dataPointFeed = options?.dataPointFeed ??
186
+ createGatewayDataPointFeed({ gatewayUrl: config.gateway.url });
187
+ // Read-side memory of tombstones, shared by the sync workers (which feed
188
+ // it) and the data route (which consults it on every read).
189
+ const scopeDeletions = createScopeDeletionTracker({
190
+ feed: dataPointFeed,
191
+ serverOwner,
192
+ logger,
193
+ });
194
+ // --- Derivative compute (question -> derivative) ---
195
+ // The scheduler exists before the sync manager so downloads can mark
196
+ // questions stale; it reaches the sync manager lazily to upload results.
197
+ const questionStore = createSqliteQuestionStore(db);
198
+ const inferenceBaseUrl = process.env.INFERENCE_BASE_URL ?? config.inference.baseUrl;
199
+ // E2EE to the Phala gateway is on unless config or INFERENCE_E2EE=false
200
+ // turns it off (local development against a provider without ACI).
201
+ const inferenceE2ee = process.env.INFERENCE_E2EE !== undefined
202
+ ? process.env.INFERENCE_E2EE !== "false"
203
+ : config.inference.e2ee;
204
+ // Local development only: production relays hold the provider key. A key
205
+ // means the base URL is a provider, not the relay, so nothing is signed.
206
+ const inferenceApiKey = process.env.INFERENCE_API_KEY;
207
+ // The Vana inference relay only forwards requests signed by the owner or
208
+ // by one of the owner's active registered servers; this is the same
209
+ // server-key signer the lineage reads use.
210
+ const inferenceSigner = inferenceApiKey ? undefined : requestSigner;
211
+ const inferenceProvider = options?.inferenceProvider ??
212
+ createOpenAiCompatibleInferenceProvider({
213
+ baseUrl: inferenceBaseUrl,
214
+ model: process.env.INFERENCE_MODEL ?? config.inference.model,
215
+ apiKey: inferenceApiKey,
216
+ requestSigner: inferenceSigner,
217
+ encryption: inferenceE2ee
218
+ ? createPhalaE2eeEncryption({
219
+ baseUrl: inferenceBaseUrl,
220
+ requestSigner: inferenceSigner,
221
+ logger,
222
+ })
223
+ : undefined,
224
+ });
225
+ if (!inferenceE2ee) {
226
+ logger.warn({ baseUrl: inferenceBaseUrl }, "Inference E2EE is disabled: prompts and answers travel as plaintext over TLS");
227
+ }
228
+ const derivativeScheduler = createRecomputeScheduler({
229
+ store: questionStore,
230
+ debounceMs: config.inference.recomputeDebounceMs,
231
+ serverOwner,
232
+ logger,
233
+ compute: (questionId) => computeQuestion(questionId, {
234
+ // A -> B -> C: a question reading this derived scope recomputes.
235
+ onDerivedWritten: (event) => derivativeScheduler.markSourceChanged(event.scope, {
236
+ lineageSources: event.lineageSources,
237
+ }),
238
+ storage: dataStorage,
239
+ store: questionStore,
240
+ provider: inferenceProvider,
241
+ serverOwner,
242
+ maxSourceItems: config.inference.maxSourceItems,
243
+ syncManager: {
244
+ notifyNewData: () => syncManager?.notifyNewData(),
245
+ },
246
+ scopeDeletions,
247
+ writePolicyPorts: {
248
+ authSessionVerifier: gatewayClient,
249
+ grantVerifier: gatewayClient,
250
+ },
251
+ logger,
252
+ }),
253
+ });
254
+ const derivativeCompute = {
255
+ store: questionStore,
256
+ scheduler: derivativeScheduler,
257
+ };
75
258
  if (config.sync.enabled &&
76
259
  masterKeySignature &&
77
260
  serverOwner &&
78
261
  serverAccount &&
79
262
  serverSigner) {
80
263
  const masterKey = deriveMasterKey(masterKeySignature);
81
- const vanaConfig = config.storage.config.vana ?? {
82
- apiUrl: "https://storage.vana.com",
83
- };
84
- const requestSigner = createRequestSigner(serverAccount);
85
- const storageAdapter = createVanaStorageAdapter({
86
- apiUrl: vanaConfig.apiUrl,
87
- ownerAddress: serverOwner,
88
- signer: requestSigner,
264
+ const storageAdapter = createVanaSyncStorageAdapter({
265
+ config,
266
+ serverOwner,
267
+ serverAccount,
268
+ reads: isEnclave ? "public" : "signed",
269
+ });
270
+ const cursor = createSyncCursor(syncCursorPath, {
271
+ legacyConfigPath: configPath,
89
272
  });
90
- const cursor = createSyncCursor(configPath);
91
273
  const uploadDeps = {
92
- indexManager,
93
- hierarchyOptions,
274
+ storage: dataStorage,
94
275
  storageAdapter,
95
276
  gateway: gatewayClient,
96
277
  signer: serverSigner,
97
278
  masterKey,
98
279
  serverOwner,
99
280
  logger,
281
+ lineageGateway,
282
+ dataPointFeed,
283
+ scopeDeletions,
100
284
  };
101
285
  const downloadDeps = {
102
- indexManager,
103
- hierarchyOptions,
286
+ storage: dataStorage,
104
287
  storageAdapter,
105
288
  gateway: gatewayClient,
106
289
  cursor,
107
290
  masterKey,
108
291
  serverOwner,
109
292
  logger,
293
+ dataPointFeed,
294
+ scopeDeletions,
295
+ onDataPointIndexed: (event) => derivativeScheduler.markSourceChanged(event.scope, {
296
+ lineageSources: event.lineageSources,
297
+ }),
110
298
  };
111
- syncManager = createSyncManager(uploadDeps, downloadDeps);
299
+ // Durable deletion: gateway tombstone signed with the same server
300
+ // account + AddData signer uploads use, storage DELETE authorised with
301
+ // the same Web3Signed signer the storage provider uploads with.
302
+ const deleteData = isEnclave
303
+ ? null
304
+ : createGatewayDeleteDataPort({
305
+ gatewayUrl: config.gateway.url,
306
+ dataPointFeed,
307
+ serverOwner,
308
+ signer: serverSigner,
309
+ storage: {
310
+ endpoint: resolveVanaStorageEndpoint(config),
311
+ chainId: config.gateway.chainId,
312
+ signMessage: (message) => serverAccount.signMessage(message),
313
+ },
314
+ });
315
+ const pendingBlobDeletions = createFilePendingBlobDeletionStore(pendingBlobDeletionsPath);
316
+ syncManager = createSyncManager(uploadDeps, downloadDeps, {
317
+ deleteData,
318
+ pendingBlobDeletions,
319
+ transferMode: isEnclave ? "download-only" : "upload-download",
320
+ async canSync() {
321
+ try {
322
+ const serverInfo = await gatewayClient.getServer(serverAccount.address);
323
+ identity.serverId = serverInfo?.id ?? null;
324
+ if (serverInfo?.id)
325
+ return { ok: true };
326
+ return {
327
+ ok: false,
328
+ reason: "unregistered",
329
+ message: "Register this Personal Server before syncing.",
330
+ };
331
+ }
332
+ catch (err) {
333
+ logger.warn({ err }, "Could not verify server registration for sync");
334
+ return {
335
+ ok: false,
336
+ reason: "registration_check_failed",
337
+ message: "Could not verify server registration before syncing.",
338
+ };
339
+ }
340
+ },
341
+ });
112
342
  syncManager.start();
113
343
  logger.info("Sync engine started");
114
344
  }
@@ -120,11 +350,51 @@ export async function createServer(config, options) {
120
350
  const accessLogWriter = createAccessLogWriter(logsDir);
121
351
  const accessLogReader = createAccessLogReader(logsDir);
122
352
  // Generate ephemeral dev token when devUi is enabled
123
- const devToken = config.devUi.enabled ? generateDevToken() : undefined;
353
+ const devToken = config.devUi.enabled && !isEnclave ? generateDevToken() : undefined;
354
+ // Token store for /auth/device flow (self-hosted CLI auth)
355
+ const tokenStore = createTokenStore(tokensPath, logger);
356
+ const cloudMode = process.env.CLOUD_MODE === "true";
357
+ const localApprovalPort = cloudMode || isEnclave
358
+ ? undefined
359
+ : resolveLocalApprovalPort(config.server.port);
124
360
  // Mutable origin — starts with config value, updated when tunnel connects
125
361
  let effectiveOrigin = config.server.origin;
362
+ let effectiveLocalApprovalOrigin;
126
363
  // Mutable tunnelManager — set when tunnel starts in background
127
364
  let tunnelManager;
365
+ // Registration can land after boot (desktop signing exchange or the
366
+ // /ui/api registration route); the tunnel connect is gated on it, so the
367
+ // route needs a way to trigger the deferred connect. Assigned inside
368
+ // startBackgroundServices once the tunnel is reserved.
369
+ let notifyServerRegistered = () => { };
370
+ let registrationPollTimer = null;
371
+ // Set by cleanup(). A poll iteration that is mid-flight when cleanup runs
372
+ // holds no timer handle, so clearing the timer alone cannot stop it — and
373
+ // a reserved-but-never-connected TunnelManager would happily spawn frpc
374
+ // from an orphaned poll after shutdown. Every deferred path checks this.
375
+ let backgroundClosed = false;
376
+ const clearRegistrationPoll = () => {
377
+ if (registrationPollTimer) {
378
+ clearTimeout(registrationPollTimer);
379
+ registrationPollTimer = null;
380
+ }
381
+ };
382
+ const ecies = new NodeECIESProvider();
383
+ const jobWorker = isEnclave && serverOwner && serverAccount
384
+ ? (envelope) => executeJob(envelope, {
385
+ serverOwner,
386
+ serverAddress: serverAccount.address,
387
+ serverPublicKey: serverAccount.publicKey,
388
+ authAudience: config.gateway.url.replace(TRAILING_SLASHES, ""),
389
+ gateway: gatewayClient,
390
+ gatewayConfig: config.gateway,
391
+ storage: dataStorage,
392
+ ecies,
393
+ logger,
394
+ accessLogWriter,
395
+ scopeDeletions,
396
+ })
397
+ : undefined;
128
398
  const app = createApp({
129
399
  logger,
130
400
  version: pkg.version,
@@ -132,18 +402,40 @@ export async function createServer(config, options) {
132
402
  indexManager,
133
403
  hierarchyOptions,
134
404
  serverOrigin: () => effectiveOrigin,
405
+ localApprovalOrigin: () => effectiveLocalApprovalOrigin,
135
406
  serverOwner,
136
407
  identity,
137
408
  gateway: gatewayClient,
409
+ gatewayConfig: config.gateway,
410
+ paymentEnabled: config.payment.enabled,
411
+ gatewayUrl: config.gateway.url,
412
+ lineageGateway,
413
+ derivativeCompute,
414
+ config,
138
415
  accessLogWriter,
139
416
  accessLogReader,
417
+ readFulfillmentReporter: options?.readFulfillmentReporter,
418
+ dataStorage,
419
+ scopeDeletions,
420
+ cloudMode,
140
421
  devToken,
422
+ ownerSignature: masterKeySignature,
423
+ ownerPrivateKey,
424
+ accessToken,
425
+ tokenStore,
141
426
  configPath,
142
427
  syncManager,
143
428
  serverSigner,
144
429
  getTunnelStatus: () => tunnelManager?.getStatus() ?? null,
430
+ mcpOAuthApprovalUrl: options?.mcpOAuthApprovalUrl,
431
+ onServerRegistered: (serverId) => notifyServerRegistered(serverId),
432
+ profile: options?.profile,
433
+ jobWorker,
145
434
  });
146
435
  const cleanup = async () => {
436
+ backgroundClosed = true;
437
+ clearRegistrationPoll();
438
+ derivativeScheduler.stop();
147
439
  if (tunnelManager) {
148
440
  await tunnelManager.stop();
149
441
  }
@@ -165,8 +457,20 @@ export async function createServer(config, options) {
165
457
  syncManager,
166
458
  tunnelManager,
167
459
  tunnelUrl: undefined,
460
+ isServerRegistered: () => Boolean(identity?.serverId),
461
+ localApprovalPort,
462
+ getLocalApprovalOrigin: () => effectiveLocalApprovalOrigin,
463
+ setLocalApprovalOrigin: (origin) => {
464
+ effectiveLocalApprovalOrigin = origin;
465
+ },
168
466
  devToken,
169
467
  startBackgroundServices: async () => {
468
+ // Replay questions a previous run left pending or stale: nothing
469
+ // else reschedules them after a restart (registration and source
470
+ // changes only cover new activity).
471
+ if (!isEnclave) {
472
+ derivativeScheduler.start();
473
+ }
170
474
  // --- Gateway registration check (slow: HTTP call) ---
171
475
  if (serverAccount && identity) {
172
476
  try {
@@ -187,33 +491,124 @@ export async function createServer(config, options) {
187
491
  }
188
492
  }
189
493
  // --- Tunnel setup (slow: subprocess wait) ---
190
- if (config.tunnel.enabled &&
494
+ if (!isEnclave &&
495
+ config.tunnel.enabled &&
191
496
  serverOwner &&
192
497
  serverAccount &&
193
498
  frpcBinaryPath) {
194
499
  tunnelManager = new TunnelManager(storageRoot);
195
500
  context.tunnelManager = tunnelManager;
196
501
  const runId = randomUUID();
502
+ let tunnelConnectStarted = false;
503
+ const connectTunnel = async () => {
504
+ const manager = tunnelManager;
505
+ if (!manager || tunnelConnectStarted || backgroundClosed) {
506
+ return;
507
+ }
508
+ tunnelConnectStarted = true;
509
+ try {
510
+ const url = await manager.connect();
511
+ const tunnelStatus = manager.getStatus();
512
+ if (tunnelStatus.status === "connected") {
513
+ logger.info({ tunnelUrl: url }, "Tunnel established");
514
+ }
515
+ else {
516
+ logger.warn({ tunnelUrl: url, warning: tunnelStatus.warning }, "Tunnel URL reserved; waiting for tunnel connection");
517
+ }
518
+ }
519
+ catch (err) {
520
+ logger.warn({ err }, "Tunnel failed to connect - server running in local-only mode");
521
+ tunnelManager = undefined;
522
+ context.tunnelManager = undefined;
523
+ // Local-only mode must keep the local audience/URL: leaving the
524
+ // public URL as effectiveOrigin would reject local Web3Signed
525
+ // callers against an origin that will never route.
526
+ effectiveOrigin = config.server.origin;
527
+ context.tunnelUrl = undefined;
528
+ }
529
+ };
530
+ // Poll fallback for registrations the route callback can't see
531
+ // (e.g. desktop registers with the gateway directly and the sidecar
532
+ // only observes it via a gateway lookup).
533
+ const scheduleRegistrationPoll = () => {
534
+ const pollStartedAt = Date.now();
535
+ const poll = async () => {
536
+ registrationPollTimer = null;
537
+ if (backgroundClosed) {
538
+ return;
539
+ }
540
+ try {
541
+ const serverInfo = await gatewayClient.getServer(serverAccount.address);
542
+ if (serverInfo?.id) {
543
+ if (identity) {
544
+ identity.serverId = serverInfo.id;
545
+ }
546
+ logger.info("Server registration detected — starting tunnel");
547
+ await connectTunnel();
548
+ return;
549
+ }
550
+ }
551
+ catch {
552
+ // Gateway unreachable — keep polling.
553
+ }
554
+ if (backgroundClosed) {
555
+ return;
556
+ }
557
+ const interval = Date.now() - pollStartedAt < 2 * 60_000 ? 5_000 : 30_000;
558
+ registrationPollTimer = setTimeout(() => void poll(), interval);
559
+ registrationPollTimer.unref?.();
560
+ };
561
+ registrationPollTimer = setTimeout(() => void poll(), 5_000);
562
+ registrationPollTimer.unref?.();
563
+ };
564
+ notifyServerRegistered = (serverId) => {
565
+ if (backgroundClosed) {
566
+ return;
567
+ }
568
+ if (serverId) {
569
+ if (identity) {
570
+ identity.serverId = serverId;
571
+ }
572
+ // Only a confirmed serverId stops the poll: on a null result the
573
+ // poll keeps running so identity/health converge once the
574
+ // gateway lookup sees the registration.
575
+ clearRegistrationPoll();
576
+ }
577
+ void connectTunnel();
578
+ };
197
579
  try {
198
- const url = await tunnelManager.start({
580
+ // Reserve the public URL without dialing the relay: the URL is
581
+ // what registration needs, and dialing before the registration
582
+ // lands poisons the relay's delegation cache (BUI-611).
583
+ const url = tunnelManager.reserve({
199
584
  walletAddress: serverAccount.address,
200
585
  ownerAddress: serverOwner,
201
586
  serverKeypair: serverAccount,
587
+ tunnelSigner: ownerTunnelSigner
588
+ ? {
589
+ signMessage: (message) => ownerTunnelSigner.signMessage({ message }),
590
+ }
591
+ : undefined,
202
592
  runId,
203
593
  serverAddr: config.tunnel.serverAddr,
204
594
  serverPort: config.tunnel.serverPort,
205
595
  localPort: config.server.port,
206
596
  }, frpcBinaryPath);
207
- logger.info({ tunnelUrl: url }, "Tunnel established");
208
597
  context.tunnelUrl = url;
598
+ // The public URL is the audience clients sign against, so adopt it
599
+ // as the effective origin as soon as it's known (matches the
600
+ // unity-surfaces #583 sidecar patch).
209
601
  effectiveOrigin = url;
210
- if (!identity?.serverId) {
211
- logger.warn("Tunnel started but server is not registered with gateway — tunnel will not route traffic. Run: npm run register-server");
212
- tunnelManager.setVerified(false, "Server not registered with gateway");
602
+ if (identity?.serverId) {
603
+ await connectTunnel();
604
+ }
605
+ else {
606
+ logger.info({ tunnelUrl: url }, "Tunnel start deferred until server registration — register to enable routing");
607
+ scheduleRegistrationPoll();
213
608
  }
214
609
  }
215
610
  catch (err) {
216
- logger.warn({ err }, "Tunnel failed to connect - server running in local-only mode");
611
+ logger.warn({ err }, "Tunnel failed to start - server running in local-only mode");
217
612
  tunnelManager = undefined;
218
613
  context.tunnelManager = undefined;
219
614
  }