@opendatalabs/personal-server-ts-server 0.0.1-canary.016adf9 → 0.0.1-canary.052d47e
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/api-auth.d.ts +31 -0
- package/dist/api-auth.d.ts.map +1 -0
- package/dist/api-auth.js +148 -0
- package/dist/api-auth.js.map +1 -0
- package/dist/api.d.ts +4 -0
- package/dist/api.d.ts.map +1 -1
- package/dist/api.js +3 -0
- package/dist/api.js.map +1 -1
- package/dist/app.d.ts +77 -2
- package/dist/app.d.ts.map +1 -1
- package/dist/app.js +165 -9
- package/dist/app.js.map +1 -1
- package/dist/bootstrap.d.ts +36 -3
- package/dist/bootstrap.d.ts.map +1 -1
- package/dist/bootstrap.js +434 -39
- package/dist/bootstrap.js.map +1 -1
- package/dist/client.d.ts +17 -0
- package/dist/client.d.ts.map +1 -0
- package/dist/client.js +369 -0
- package/dist/client.js.map +1 -0
- package/dist/config/defaults.d.ts +12 -0
- package/dist/config/defaults.d.ts.map +1 -0
- package/dist/config/defaults.js +14 -0
- package/dist/config/defaults.js.map +1 -0
- package/dist/config/index.d.ts +4 -0
- package/dist/config/index.d.ts.map +1 -0
- package/dist/config/index.js +4 -0
- package/dist/config/index.js.map +1 -0
- package/dist/config/loader.d.ts +8 -0
- package/dist/config/loader.d.ts.map +1 -0
- package/dist/config/loader.js +78 -0
- package/dist/config/loader.js.map +1 -0
- package/dist/config/paths.d.ts +9 -0
- package/dist/config/paths.d.ts.map +1 -0
- package/dist/config/paths.js +22 -0
- package/dist/config/paths.js.map +1 -0
- package/dist/enclave-main.d.ts +10 -0
- package/dist/enclave-main.d.ts.map +1 -0
- package/dist/enclave-main.js +115 -0
- package/dist/enclave-main.js.map +1 -0
- package/dist/hono-context.d.ts +14 -0
- package/dist/hono-context.d.ts.map +1 -0
- package/dist/hono-context.js +2 -0
- package/dist/hono-context.js.map +1 -0
- package/dist/index.js +48 -12
- package/dist/index.js.map +1 -1
- package/dist/jobs/types.d.ts +17 -0
- package/dist/jobs/types.d.ts.map +1 -0
- package/dist/jobs/types.js +3 -0
- package/dist/jobs/types.js.map +1 -0
- package/dist/jobs/worker.d.ts +30 -0
- package/dist/jobs/worker.d.ts.map +1 -0
- package/dist/jobs/worker.js +217 -0
- package/dist/jobs/worker.js.map +1 -0
- package/dist/keys/server-account.d.ts +28 -0
- package/dist/keys/server-account.d.ts.map +1 -0
- package/dist/keys/server-account.js +48 -0
- package/dist/keys/server-account.js.map +1 -0
- package/dist/listen.d.ts +11 -0
- package/dist/listen.d.ts.map +1 -0
- package/dist/listen.js +23 -0
- package/dist/listen.js.map +1 -0
- package/dist/logger/index.d.ts +5 -0
- package/dist/logger/index.d.ts.map +1 -0
- package/dist/logger/index.js +9 -0
- package/dist/logger/index.js.map +1 -0
- package/dist/logging/access-log.d.ts +4 -0
- package/dist/logging/access-log.d.ts.map +1 -0
- package/dist/logging/access-log.js +25 -0
- package/dist/logging/access-log.js.map +1 -0
- package/dist/logging/access-reader.d.ts +19 -0
- package/dist/logging/access-reader.d.ts.map +1 -0
- package/dist/logging/access-reader.js +50 -0
- package/dist/logging/access-reader.js.map +1 -0
- package/dist/middleware/access-log.d.ts.map +1 -1
- package/dist/middleware/access-log.js +8 -2
- package/dist/middleware/access-log.js.map +1 -1
- package/dist/middleware/builder-check.d.ts +2 -2
- package/dist/middleware/builder-check.d.ts.map +1 -1
- package/dist/middleware/builder-check.js +7 -2
- package/dist/middleware/builder-check.js.map +1 -1
- package/dist/middleware/cors.d.ts +18 -0
- package/dist/middleware/cors.d.ts.map +1 -0
- package/dist/middleware/cors.js +51 -0
- package/dist/middleware/cors.js.map +1 -0
- package/dist/middleware/owner-check.js +1 -1
- package/dist/middleware/owner-check.js.map +1 -1
- package/dist/middleware/web3-auth.d.ts +12 -0
- package/dist/middleware/web3-auth.d.ts.map +1 -1
- package/dist/middleware/web3-auth.js +27 -31
- package/dist/middleware/web3-auth.js.map +1 -1
- package/dist/migrations/detect-stuck-version-ledger.d.ts +43 -0
- package/dist/migrations/detect-stuck-version-ledger.d.ts.map +1 -0
- package/dist/migrations/detect-stuck-version-ledger.js +73 -0
- package/dist/migrations/detect-stuck-version-ledger.js.map +1 -0
- package/dist/migrations/local-state.d.ts +35 -0
- package/dist/migrations/local-state.d.ts.map +1 -0
- package/dist/migrations/local-state.js +133 -0
- package/dist/migrations/local-state.js.map +1 -0
- package/dist/migrations/state-migrations.d.ts +78 -0
- package/dist/migrations/state-migrations.d.ts.map +1 -0
- package/dist/migrations/state-migrations.js +72 -0
- package/dist/migrations/state-migrations.js.map +1 -0
- package/dist/pending-blob-deletions.d.ts +10 -0
- package/dist/pending-blob-deletions.d.ts.map +1 -0
- package/dist/pending-blob-deletions.js +38 -0
- package/dist/pending-blob-deletions.js.map +1 -0
- package/dist/routes/access-logs.d.ts +7 -0
- package/dist/routes/access-logs.d.ts.map +1 -1
- package/dist/routes/access-logs.js +11 -17
- package/dist/routes/access-logs.js.map +1 -1
- package/dist/routes/auth-device.d.ts +49 -0
- package/dist/routes/auth-device.d.ts.map +1 -0
- package/dist/routes/auth-device.js +581 -0
- package/dist/routes/auth-device.js.map +1 -0
- package/dist/routes/data.d.ts +67 -2
- package/dist/routes/data.d.ts.map +1 -1
- package/dist/routes/data.js +41 -201
- package/dist/routes/data.js.map +1 -1
- package/dist/routes/derivatives.d.ts +33 -0
- package/dist/routes/derivatives.d.ts.map +1 -0
- package/dist/routes/derivatives.js +33 -0
- package/dist/routes/derivatives.js.map +1 -0
- package/dist/routes/enclave-jobs.d.ts +9 -0
- package/dist/routes/enclave-jobs.d.ts.map +1 -0
- package/dist/routes/enclave-jobs.js +104 -0
- package/dist/routes/enclave-jobs.js.map +1 -0
- package/dist/routes/grants.d.ts +9 -2
- package/dist/routes/grants.d.ts.map +1 -1
- package/dist/routes/grants.js +18 -196
- package/dist/routes/grants.js.map +1 -1
- package/dist/routes/health.d.ts +7 -1
- package/dist/routes/health.d.ts.map +1 -1
- package/dist/routes/health.js +39 -2
- package/dist/routes/health.js.map +1 -1
- package/dist/routes/mcp.d.ts +115 -0
- package/dist/routes/mcp.d.ts.map +1 -0
- package/dist/routes/mcp.js +676 -0
- package/dist/routes/mcp.js.map +1 -0
- package/dist/routes/oauth-token.d.ts +56 -0
- package/dist/routes/oauth-token.d.ts.map +1 -0
- package/dist/routes/oauth-token.js +49 -0
- package/dist/routes/oauth-token.js.map +1 -0
- package/dist/routes/sync.d.ts +7 -0
- package/dist/routes/sync.d.ts.map +1 -1
- package/dist/routes/sync.js +12 -39
- package/dist/routes/sync.js.map +1 -1
- package/dist/routes/ui-config.d.ts.map +1 -1
- package/dist/routes/ui-config.js +17 -82
- package/dist/routes/ui-config.js.map +1 -1
- package/dist/routes/ui-registration.d.ts +9 -0
- package/dist/routes/ui-registration.d.ts.map +1 -0
- package/dist/routes/ui-registration.js +217 -0
- package/dist/routes/ui-registration.js.map +1 -0
- package/dist/routes/ui.d.ts +1 -0
- package/dist/routes/ui.d.ts.map +1 -1
- package/dist/routes/ui.js +46 -4
- package/dist/routes/ui.js.map +1 -1
- package/dist/routes/write-session.d.ts +37 -0
- package/dist/routes/write-session.d.ts.map +1 -0
- package/dist/routes/write-session.js +34 -0
- package/dist/routes/write-session.js.map +1 -0
- package/dist/storage/hierarchy.d.ts +36 -0
- package/dist/storage/hierarchy.d.ts.map +1 -0
- package/dist/storage/hierarchy.js +282 -0
- package/dist/storage/hierarchy.js.map +1 -0
- package/dist/storage/index-manager.d.ts +60 -0
- package/dist/storage/index-manager.d.ts.map +1 -0
- package/dist/storage/index-manager.js +174 -0
- package/dist/storage/index-manager.js.map +1 -0
- package/dist/storage/index-schema.d.ts +5 -0
- package/dist/storage/index-schema.d.ts.map +1 -0
- package/dist/storage/index-schema.js +69 -0
- package/dist/storage/index-schema.js.map +1 -0
- package/dist/storage/node-data-storage.d.ts +9 -0
- package/dist/storage/node-data-storage.d.ts.map +1 -0
- package/dist/storage/node-data-storage.js +105 -0
- package/dist/storage/node-data-storage.js.map +1 -0
- package/dist/storage/question-store.d.ts +9 -0
- package/dist/storage/question-store.d.ts.map +1 -0
- package/dist/storage/question-store.js +151 -0
- package/dist/storage/question-store.js.map +1 -0
- package/dist/sync-cursor.d.ts +15 -0
- package/dist/sync-cursor.d.ts.map +1 -0
- package/dist/sync-cursor.js +110 -0
- package/dist/sync-cursor.js.map +1 -0
- package/dist/token-store.d.ts +22 -0
- package/dist/token-store.d.ts.map +1 -0
- package/dist/token-store.js +120 -0
- package/dist/token-store.js.map +1 -0
- package/dist/tunnel/auth.d.ts +4 -0
- package/dist/tunnel/auth.d.ts.map +1 -1
- package/dist/tunnel/auth.js +2 -1
- package/dist/tunnel/auth.js.map +1 -1
- package/dist/tunnel/config.d.ts +1 -0
- package/dist/tunnel/config.d.ts.map +1 -1
- package/dist/tunnel/config.js +1 -1
- package/dist/tunnel/config.js.map +1 -1
- package/dist/tunnel/manager.d.ts +21 -1
- package/dist/tunnel/manager.d.ts.map +1 -1
- package/dist/tunnel/manager.js +78 -23
- package/dist/tunnel/manager.js.map +1 -1
- package/dist/ui/browser_tls_rustls_bg.wasm +0 -0
- package/dist/ui/index.html +2192 -0
- package/dist/ui/ps-lite-debug.d.ts +93 -0
- package/dist/ui/ps-lite-debug.d.ts.map +1 -0
- package/dist/ui/ps-lite-debug.js +135347 -0
- package/dist/ui/ps-lite-debug.js.map +1 -0
- package/package.json +20 -6
- package/dist/middleware/grant-check.d.ts +0 -12
- package/dist/middleware/grant-check.d.ts.map +0 -1
- package/dist/middleware/grant-check.js +0 -90
- package/dist/middleware/grant-check.js.map +0 -1
package/dist/bootstrap.js
CHANGED
|
@@ -2,48 +2,118 @@ import { mkdir } from "node:fs/promises";
|
|
|
2
2
|
import { createRequire } from "node:module";
|
|
3
3
|
import { randomUUID } from "node:crypto";
|
|
4
4
|
import { join } from "node:path";
|
|
5
|
+
import { privateKeyToAccount } from "viem/accounts";
|
|
5
6
|
const require = createRequire(import.meta.url);
|
|
6
7
|
const pkg = require("../package.json");
|
|
7
|
-
import { DEFAULT_ROOT_PATH, resolveRootPath
|
|
8
|
-
import { createLogger
|
|
9
|
-
import { initializeDatabase
|
|
10
|
-
import {
|
|
11
|
-
import {
|
|
12
|
-
import {
|
|
13
|
-
import {
|
|
14
|
-
import {
|
|
15
|
-
import {
|
|
16
|
-
import {
|
|
8
|
+
import { DEFAULT_ROOT_PATH, resolveRootPath } from "./config/index.js";
|
|
9
|
+
import { createLogger } from "./logger/index.js";
|
|
10
|
+
import { initializeDatabase } from "./storage/index-schema.js";
|
|
11
|
+
import { createIndexManager, } from "./storage/index-manager.js";
|
|
12
|
+
import { createGatewayClient, NodeECIESProvider, } from "@opendatalabs/vana-sdk/node";
|
|
13
|
+
import { createAccessLogWriter } from "./logging/access-log.js";
|
|
14
|
+
import { createAccessLogReader } from "./logging/access-reader.js";
|
|
15
|
+
import { deriveMasterKey, recoverServerOwner, } from "@opendatalabs/vana-sdk/node";
|
|
16
|
+
import { loadOrCreateServerAccount } from "./keys/server-account.js";
|
|
17
|
+
import { createRequestSigner, createServerSigner, } from "@opendatalabs/personal-server-ts-core/signing";
|
|
18
|
+
import { createGatewayLineageClient, } from "@opendatalabs/personal-server-ts-core/lineage";
|
|
19
|
+
import { createSyncCursor } from "./sync-cursor.js";
|
|
20
|
+
import { createGatewayDataPointFeed, createGatewayDeleteDataPort, createScopeDeletionTracker, createSyncManager, } from "@opendatalabs/personal-server-ts-core/sync";
|
|
21
|
+
import { createVanaSyncStorageAdapter, resolveVanaStorageEndpoint, } from "@opendatalabs/personal-server-ts-core/storage/adapters";
|
|
22
|
+
import { createFilePendingBlobDeletionStore } from "./pending-blob-deletions.js";
|
|
17
23
|
import { createApp } from "./app.js";
|
|
18
24
|
import { generateDevToken } from "./dev-token.js";
|
|
25
|
+
import { migrateLocalState } from "./migrations/local-state.js";
|
|
26
|
+
import { createTokenStore } from "./token-store.js";
|
|
19
27
|
import { TunnelManager, ensureFrpcBinary } from "./tunnel/index.js";
|
|
28
|
+
import { createNodeDataStorage } from "./storage/node-data-storage.js";
|
|
29
|
+
import { createSqliteQuestionStore } from "./storage/question-store.js";
|
|
30
|
+
import { computeQuestion, createOpenAiCompatibleInferenceProvider, createPhalaE2eeEncryption, createRecomputeScheduler, } from "@opendatalabs/personal-server-ts-core/derivatives";
|
|
31
|
+
import { executeJob } from "./jobs/worker.js";
|
|
32
|
+
const TRAILING_SLASHES = /\/+$/;
|
|
33
|
+
const DEFAULT_LOCAL_APPROVAL_PORT = 34127;
|
|
34
|
+
function resolveLocalApprovalPort(serverPort) {
|
|
35
|
+
const raw = process.env.LOCAL_AUTH_PORT;
|
|
36
|
+
if (raw !== undefined) {
|
|
37
|
+
const parsed = Number(raw);
|
|
38
|
+
if (Number.isInteger(parsed) &&
|
|
39
|
+
parsed >= 1 &&
|
|
40
|
+
parsed <= 65535 &&
|
|
41
|
+
parsed !== serverPort) {
|
|
42
|
+
return parsed;
|
|
43
|
+
}
|
|
44
|
+
}
|
|
45
|
+
const adjacentPort = serverPort + 1;
|
|
46
|
+
if (adjacentPort <= 65535) {
|
|
47
|
+
return adjacentPort;
|
|
48
|
+
}
|
|
49
|
+
return DEFAULT_LOCAL_APPROVAL_PORT;
|
|
50
|
+
}
|
|
20
51
|
export async function createServer(config, options) {
|
|
21
52
|
const logger = createLogger(config.logging);
|
|
22
53
|
const startedAt = new Date();
|
|
54
|
+
const isEnclave = options?.profile === "enclave";
|
|
55
|
+
if (isEnclave && process.env.VANA_OWNER_PRIVATE_KEY) {
|
|
56
|
+
throw new Error("VANA_OWNER_PRIVATE_KEY is forbidden in enclave profile");
|
|
57
|
+
}
|
|
23
58
|
const storageRoot = resolveRootPath(options?.rootPath ?? options?.serverDir ?? DEFAULT_ROOT_PATH);
|
|
24
59
|
const dataDir = options?.dataDir ?? join(storageRoot, "data");
|
|
25
60
|
const indexPath = join(storageRoot, "index.db");
|
|
26
61
|
const configPath = join(storageRoot, "config.json");
|
|
62
|
+
const syncCursorPath = join(storageRoot, "sync-cursor.json");
|
|
63
|
+
const pendingBlobDeletionsPath = join(storageRoot, "pending-blob-deletions.json");
|
|
64
|
+
const tokensPath = join(storageRoot, "tokens.json");
|
|
27
65
|
await mkdir(storageRoot, { recursive: true });
|
|
28
66
|
await mkdir(dataDir, { recursive: true });
|
|
29
67
|
const db = initializeDatabase(indexPath);
|
|
68
|
+
await migrateLocalState({
|
|
69
|
+
storageRoot,
|
|
70
|
+
dataDir,
|
|
71
|
+
configPath,
|
|
72
|
+
syncCursorPath,
|
|
73
|
+
tokensPath,
|
|
74
|
+
db,
|
|
75
|
+
logger,
|
|
76
|
+
});
|
|
30
77
|
const indexManager = createIndexManager(db);
|
|
31
78
|
const hierarchyOptions = { dataDir };
|
|
32
|
-
const
|
|
79
|
+
const dataStorage = createNodeDataStorage({ indexManager, hierarchyOptions });
|
|
80
|
+
const gatewayClient = options?.gatewayClient ?? createGatewayClient(config.gateway.url);
|
|
81
|
+
// X402 payment enforcement on data reads. When config.payment.enabled is
|
|
82
|
+
// true, GET /v1/data/:scope requires builders to supply a signed X-PAYMENT
|
|
83
|
+
// header on every read; the server forwards to gateway.payForOperation.
|
|
84
|
+
// See packages/core/src/payment/x402.ts. Off-by-default for development
|
|
85
|
+
// and test setups where signing payment per read is friction.
|
|
86
|
+
if (config.payment.enabled) {
|
|
87
|
+
logger.info("X402 payment enforcement enabled on data reads");
|
|
88
|
+
}
|
|
33
89
|
// Derive server owner from VANA_MASTER_KEY_SIGNATURE env var
|
|
34
|
-
const masterKeySignature =
|
|
90
|
+
const masterKeySignature = options?.ownerSignature ??
|
|
91
|
+
process.env.VANA_MASTER_KEY_SIGNATURE;
|
|
92
|
+
const ownerPrivateKey = process.env.VANA_OWNER_PRIVATE_KEY;
|
|
93
|
+
const ownerTunnelSigner = ownerPrivateKey
|
|
94
|
+
? privateKeyToAccount(ownerPrivateKey.startsWith("0x")
|
|
95
|
+
? ownerPrivateKey
|
|
96
|
+
: `0x${ownerPrivateKey}`)
|
|
97
|
+
: null;
|
|
35
98
|
let serverOwner;
|
|
36
99
|
let serverAccount;
|
|
37
100
|
let serverSigner;
|
|
38
101
|
let identity;
|
|
39
102
|
if (masterKeySignature) {
|
|
40
|
-
|
|
103
|
+
const recoveredServerOwner = await recoverServerOwner(masterKeySignature);
|
|
104
|
+
serverOwner = recoveredServerOwner;
|
|
41
105
|
deriveMasterKey(masterKeySignature); // validate signature format
|
|
42
106
|
logger.info({ owner: serverOwner }, "Server owner derived from master key");
|
|
43
|
-
|
|
107
|
+
if (ownerTunnelSigner &&
|
|
108
|
+
ownerTunnelSigner.address.toLowerCase() !==
|
|
109
|
+
recoveredServerOwner.toLowerCase()) {
|
|
110
|
+
throw new Error("VANA_OWNER_PRIVATE_KEY does not match VANA_MASTER_KEY_SIGNATURE owner");
|
|
111
|
+
}
|
|
112
|
+
// Enclaves receive public identity only; standard mode owns its key file.
|
|
44
113
|
const keyPath = join(storageRoot, "key.json");
|
|
45
|
-
serverAccount =
|
|
46
|
-
|
|
114
|
+
serverAccount =
|
|
115
|
+
options?.serverAccount ?? loadOrCreateServerAccount(keyPath);
|
|
116
|
+
logger.info({ owner: serverOwner, serverAddress: serverAccount.address }, "Server account loaded");
|
|
47
117
|
serverSigner = createServerSigner(serverAccount, {
|
|
48
118
|
chainId: config.gateway.chainId,
|
|
49
119
|
contracts: config.gateway.contracts,
|
|
@@ -58,9 +128,33 @@ export async function createServer(config, options) {
|
|
|
58
128
|
else {
|
|
59
129
|
logger.warn("VANA_MASTER_KEY_SIGNATURE not set — owner-restricted endpoints will return 500");
|
|
60
130
|
}
|
|
131
|
+
// Resolve PS_ACCESS_TOKEN for cloud control-plane / bootstrap auth.
|
|
132
|
+
// This is not the per-login CLI session token returned to users.
|
|
133
|
+
let accessToken = process.env.PS_ACCESS_TOKEN;
|
|
134
|
+
if (!accessToken && process.env.CLOUD_MODE === "true") {
|
|
135
|
+
try {
|
|
136
|
+
const metadataRes = await fetch("http://metadata.google.internal/computeMetadata/v1/instance/attributes/ps-access-token", {
|
|
137
|
+
headers: { "Metadata-Flavor": "Google" },
|
|
138
|
+
signal: AbortSignal.timeout(2000),
|
|
139
|
+
});
|
|
140
|
+
if (metadataRes.ok) {
|
|
141
|
+
const token = (await metadataRes.text()).trim();
|
|
142
|
+
if (token.length > 0) {
|
|
143
|
+
accessToken = token;
|
|
144
|
+
logger.info("PS control-plane token loaded from instance metadata");
|
|
145
|
+
}
|
|
146
|
+
}
|
|
147
|
+
}
|
|
148
|
+
catch {
|
|
149
|
+
logger.debug("Could not read PS control-plane token from instance metadata");
|
|
150
|
+
}
|
|
151
|
+
}
|
|
152
|
+
if (accessToken) {
|
|
153
|
+
logger.info("PS_ACCESS_TOKEN configured — control-plane bearer auth enabled");
|
|
154
|
+
}
|
|
61
155
|
// Download frpc binary eagerly (auth-independent) so it's ready when the user signs in
|
|
62
156
|
let frpcBinaryPath = "";
|
|
63
|
-
if (config.tunnel.enabled) {
|
|
157
|
+
if (config.tunnel.enabled && !isEnclave) {
|
|
64
158
|
try {
|
|
65
159
|
frpcBinaryPath = await ensureFrpcBinary(storageRoot, {
|
|
66
160
|
log: (msg) => logger.info(msg),
|
|
@@ -70,45 +164,181 @@ export async function createServer(config, options) {
|
|
|
70
164
|
logger.warn({ err }, "Failed to download frpc binary - tunnel disabled");
|
|
71
165
|
}
|
|
72
166
|
}
|
|
167
|
+
// Derivative data: the gateway lineage client signs lineage reads with the
|
|
168
|
+
// server key (the gateway treats a registered server as the owner's) and
|
|
169
|
+
// registers derivatives with their lineage. Needs the server account.
|
|
170
|
+
// The one server-key request signer: lineage reads, and the inference
|
|
171
|
+
// relay calls below, authenticate to the gateway with the same scheme.
|
|
172
|
+
const requestSigner = serverAccount && !isEnclave
|
|
173
|
+
? createRequestSigner(serverAccount)
|
|
174
|
+
: undefined;
|
|
175
|
+
const lineageGateway = requestSigner
|
|
176
|
+
? createGatewayLineageClient({
|
|
177
|
+
gatewayUrl: config.gateway.url,
|
|
178
|
+
requestSigner,
|
|
179
|
+
})
|
|
180
|
+
: undefined;
|
|
73
181
|
// --- Sync engine setup ---
|
|
74
182
|
let syncManager = null;
|
|
183
|
+
// Deletion-aware registry view. Independent of sync: the data route uses
|
|
184
|
+
// it to answer 410 for scopes the owner deleted even when sync is off.
|
|
185
|
+
const dataPointFeed = options?.dataPointFeed ??
|
|
186
|
+
createGatewayDataPointFeed({ gatewayUrl: config.gateway.url });
|
|
187
|
+
// Read-side memory of tombstones, shared by the sync workers (which feed
|
|
188
|
+
// it) and the data route (which consults it on every read).
|
|
189
|
+
const scopeDeletions = createScopeDeletionTracker({
|
|
190
|
+
feed: dataPointFeed,
|
|
191
|
+
serverOwner,
|
|
192
|
+
logger,
|
|
193
|
+
});
|
|
194
|
+
// --- Derivative compute (question -> derivative) ---
|
|
195
|
+
// The scheduler exists before the sync manager so downloads can mark
|
|
196
|
+
// questions stale; it reaches the sync manager lazily to upload results.
|
|
197
|
+
const questionStore = createSqliteQuestionStore(db);
|
|
198
|
+
const inferenceBaseUrl = process.env.INFERENCE_BASE_URL ?? config.inference.baseUrl;
|
|
199
|
+
// E2EE to the Phala gateway is on unless config or INFERENCE_E2EE=false
|
|
200
|
+
// turns it off (local development against a provider without ACI).
|
|
201
|
+
const inferenceE2ee = process.env.INFERENCE_E2EE !== undefined
|
|
202
|
+
? process.env.INFERENCE_E2EE !== "false"
|
|
203
|
+
: config.inference.e2ee;
|
|
204
|
+
// Local development only: production relays hold the provider key. A key
|
|
205
|
+
// means the base URL is a provider, not the relay, so nothing is signed.
|
|
206
|
+
const inferenceApiKey = process.env.INFERENCE_API_KEY;
|
|
207
|
+
// The Vana inference relay only forwards requests signed by the owner or
|
|
208
|
+
// by one of the owner's active registered servers; this is the same
|
|
209
|
+
// server-key signer the lineage reads use.
|
|
210
|
+
const inferenceSigner = inferenceApiKey ? undefined : requestSigner;
|
|
211
|
+
const inferenceProvider = options?.inferenceProvider ??
|
|
212
|
+
createOpenAiCompatibleInferenceProvider({
|
|
213
|
+
baseUrl: inferenceBaseUrl,
|
|
214
|
+
model: process.env.INFERENCE_MODEL ?? config.inference.model,
|
|
215
|
+
apiKey: inferenceApiKey,
|
|
216
|
+
requestSigner: inferenceSigner,
|
|
217
|
+
encryption: inferenceE2ee
|
|
218
|
+
? createPhalaE2eeEncryption({
|
|
219
|
+
baseUrl: inferenceBaseUrl,
|
|
220
|
+
requestSigner: inferenceSigner,
|
|
221
|
+
logger,
|
|
222
|
+
})
|
|
223
|
+
: undefined,
|
|
224
|
+
});
|
|
225
|
+
if (!inferenceE2ee) {
|
|
226
|
+
logger.warn({ baseUrl: inferenceBaseUrl }, "Inference E2EE is disabled: prompts and answers travel as plaintext over TLS");
|
|
227
|
+
}
|
|
228
|
+
const derivativeScheduler = createRecomputeScheduler({
|
|
229
|
+
store: questionStore,
|
|
230
|
+
debounceMs: config.inference.recomputeDebounceMs,
|
|
231
|
+
serverOwner,
|
|
232
|
+
logger,
|
|
233
|
+
compute: (questionId) => computeQuestion(questionId, {
|
|
234
|
+
// A -> B -> C: a question reading this derived scope recomputes.
|
|
235
|
+
onDerivedWritten: (event) => derivativeScheduler.markSourceChanged(event.scope, {
|
|
236
|
+
lineageSources: event.lineageSources,
|
|
237
|
+
}),
|
|
238
|
+
storage: dataStorage,
|
|
239
|
+
store: questionStore,
|
|
240
|
+
provider: inferenceProvider,
|
|
241
|
+
serverOwner,
|
|
242
|
+
maxSourceItems: config.inference.maxSourceItems,
|
|
243
|
+
syncManager: {
|
|
244
|
+
notifyNewData: () => syncManager?.notifyNewData(),
|
|
245
|
+
},
|
|
246
|
+
scopeDeletions,
|
|
247
|
+
writePolicyPorts: {
|
|
248
|
+
authSessionVerifier: gatewayClient,
|
|
249
|
+
grantVerifier: gatewayClient,
|
|
250
|
+
},
|
|
251
|
+
logger,
|
|
252
|
+
}),
|
|
253
|
+
});
|
|
254
|
+
const derivativeCompute = {
|
|
255
|
+
store: questionStore,
|
|
256
|
+
scheduler: derivativeScheduler,
|
|
257
|
+
};
|
|
75
258
|
if (config.sync.enabled &&
|
|
76
259
|
masterKeySignature &&
|
|
77
260
|
serverOwner &&
|
|
78
261
|
serverAccount &&
|
|
79
262
|
serverSigner) {
|
|
80
263
|
const masterKey = deriveMasterKey(masterKeySignature);
|
|
81
|
-
const
|
|
82
|
-
|
|
83
|
-
|
|
84
|
-
|
|
85
|
-
|
|
86
|
-
|
|
87
|
-
|
|
88
|
-
|
|
264
|
+
const storageAdapter = createVanaSyncStorageAdapter({
|
|
265
|
+
config,
|
|
266
|
+
serverOwner,
|
|
267
|
+
serverAccount,
|
|
268
|
+
reads: isEnclave ? "public" : "signed",
|
|
269
|
+
});
|
|
270
|
+
const cursor = createSyncCursor(syncCursorPath, {
|
|
271
|
+
legacyConfigPath: configPath,
|
|
89
272
|
});
|
|
90
|
-
const cursor = createSyncCursor(configPath);
|
|
91
273
|
const uploadDeps = {
|
|
92
|
-
|
|
93
|
-
hierarchyOptions,
|
|
274
|
+
storage: dataStorage,
|
|
94
275
|
storageAdapter,
|
|
95
276
|
gateway: gatewayClient,
|
|
96
277
|
signer: serverSigner,
|
|
97
278
|
masterKey,
|
|
98
279
|
serverOwner,
|
|
99
280
|
logger,
|
|
281
|
+
lineageGateway,
|
|
282
|
+
dataPointFeed,
|
|
283
|
+
scopeDeletions,
|
|
100
284
|
};
|
|
101
285
|
const downloadDeps = {
|
|
102
|
-
|
|
103
|
-
hierarchyOptions,
|
|
286
|
+
storage: dataStorage,
|
|
104
287
|
storageAdapter,
|
|
105
288
|
gateway: gatewayClient,
|
|
106
289
|
cursor,
|
|
107
290
|
masterKey,
|
|
108
291
|
serverOwner,
|
|
109
292
|
logger,
|
|
293
|
+
dataPointFeed,
|
|
294
|
+
scopeDeletions,
|
|
295
|
+
onDataPointIndexed: (event) => derivativeScheduler.markSourceChanged(event.scope, {
|
|
296
|
+
lineageSources: event.lineageSources,
|
|
297
|
+
}),
|
|
110
298
|
};
|
|
111
|
-
|
|
299
|
+
// Durable deletion: gateway tombstone signed with the same server
|
|
300
|
+
// account + AddData signer uploads use, storage DELETE authorised with
|
|
301
|
+
// the same Web3Signed signer the storage provider uploads with.
|
|
302
|
+
const deleteData = isEnclave
|
|
303
|
+
? null
|
|
304
|
+
: createGatewayDeleteDataPort({
|
|
305
|
+
gatewayUrl: config.gateway.url,
|
|
306
|
+
dataPointFeed,
|
|
307
|
+
serverOwner,
|
|
308
|
+
signer: serverSigner,
|
|
309
|
+
storage: {
|
|
310
|
+
endpoint: resolveVanaStorageEndpoint(config),
|
|
311
|
+
chainId: config.gateway.chainId,
|
|
312
|
+
signMessage: (message) => serverAccount.signMessage(message),
|
|
313
|
+
},
|
|
314
|
+
});
|
|
315
|
+
const pendingBlobDeletions = createFilePendingBlobDeletionStore(pendingBlobDeletionsPath);
|
|
316
|
+
syncManager = createSyncManager(uploadDeps, downloadDeps, {
|
|
317
|
+
deleteData,
|
|
318
|
+
pendingBlobDeletions,
|
|
319
|
+
transferMode: isEnclave ? "download-only" : "upload-download",
|
|
320
|
+
async canSync() {
|
|
321
|
+
try {
|
|
322
|
+
const serverInfo = await gatewayClient.getServer(serverAccount.address);
|
|
323
|
+
identity.serverId = serverInfo?.id ?? null;
|
|
324
|
+
if (serverInfo?.id)
|
|
325
|
+
return { ok: true };
|
|
326
|
+
return {
|
|
327
|
+
ok: false,
|
|
328
|
+
reason: "unregistered",
|
|
329
|
+
message: "Register this Personal Server before syncing.",
|
|
330
|
+
};
|
|
331
|
+
}
|
|
332
|
+
catch (err) {
|
|
333
|
+
logger.warn({ err }, "Could not verify server registration for sync");
|
|
334
|
+
return {
|
|
335
|
+
ok: false,
|
|
336
|
+
reason: "registration_check_failed",
|
|
337
|
+
message: "Could not verify server registration before syncing.",
|
|
338
|
+
};
|
|
339
|
+
}
|
|
340
|
+
},
|
|
341
|
+
});
|
|
112
342
|
syncManager.start();
|
|
113
343
|
logger.info("Sync engine started");
|
|
114
344
|
}
|
|
@@ -120,11 +350,51 @@ export async function createServer(config, options) {
|
|
|
120
350
|
const accessLogWriter = createAccessLogWriter(logsDir);
|
|
121
351
|
const accessLogReader = createAccessLogReader(logsDir);
|
|
122
352
|
// Generate ephemeral dev token when devUi is enabled
|
|
123
|
-
const devToken = config.devUi.enabled ? generateDevToken() : undefined;
|
|
353
|
+
const devToken = config.devUi.enabled && !isEnclave ? generateDevToken() : undefined;
|
|
354
|
+
// Token store for /auth/device flow (self-hosted CLI auth)
|
|
355
|
+
const tokenStore = createTokenStore(tokensPath, logger);
|
|
356
|
+
const cloudMode = process.env.CLOUD_MODE === "true";
|
|
357
|
+
const localApprovalPort = cloudMode || isEnclave
|
|
358
|
+
? undefined
|
|
359
|
+
: resolveLocalApprovalPort(config.server.port);
|
|
124
360
|
// Mutable origin — starts with config value, updated when tunnel connects
|
|
125
361
|
let effectiveOrigin = config.server.origin;
|
|
362
|
+
let effectiveLocalApprovalOrigin;
|
|
126
363
|
// Mutable tunnelManager — set when tunnel starts in background
|
|
127
364
|
let tunnelManager;
|
|
365
|
+
// Registration can land after boot (desktop signing exchange or the
|
|
366
|
+
// /ui/api registration route); the tunnel connect is gated on it, so the
|
|
367
|
+
// route needs a way to trigger the deferred connect. Assigned inside
|
|
368
|
+
// startBackgroundServices once the tunnel is reserved.
|
|
369
|
+
let notifyServerRegistered = () => { };
|
|
370
|
+
let registrationPollTimer = null;
|
|
371
|
+
// Set by cleanup(). A poll iteration that is mid-flight when cleanup runs
|
|
372
|
+
// holds no timer handle, so clearing the timer alone cannot stop it — and
|
|
373
|
+
// a reserved-but-never-connected TunnelManager would happily spawn frpc
|
|
374
|
+
// from an orphaned poll after shutdown. Every deferred path checks this.
|
|
375
|
+
let backgroundClosed = false;
|
|
376
|
+
const clearRegistrationPoll = () => {
|
|
377
|
+
if (registrationPollTimer) {
|
|
378
|
+
clearTimeout(registrationPollTimer);
|
|
379
|
+
registrationPollTimer = null;
|
|
380
|
+
}
|
|
381
|
+
};
|
|
382
|
+
const ecies = new NodeECIESProvider();
|
|
383
|
+
const jobWorker = isEnclave && serverOwner && serverAccount
|
|
384
|
+
? (envelope) => executeJob(envelope, {
|
|
385
|
+
serverOwner,
|
|
386
|
+
serverAddress: serverAccount.address,
|
|
387
|
+
serverPublicKey: serverAccount.publicKey,
|
|
388
|
+
authAudience: config.gateway.url.replace(TRAILING_SLASHES, ""),
|
|
389
|
+
gateway: gatewayClient,
|
|
390
|
+
gatewayConfig: config.gateway,
|
|
391
|
+
storage: dataStorage,
|
|
392
|
+
ecies,
|
|
393
|
+
logger,
|
|
394
|
+
accessLogWriter,
|
|
395
|
+
scopeDeletions,
|
|
396
|
+
})
|
|
397
|
+
: undefined;
|
|
128
398
|
const app = createApp({
|
|
129
399
|
logger,
|
|
130
400
|
version: pkg.version,
|
|
@@ -132,18 +402,40 @@ export async function createServer(config, options) {
|
|
|
132
402
|
indexManager,
|
|
133
403
|
hierarchyOptions,
|
|
134
404
|
serverOrigin: () => effectiveOrigin,
|
|
405
|
+
localApprovalOrigin: () => effectiveLocalApprovalOrigin,
|
|
135
406
|
serverOwner,
|
|
136
407
|
identity,
|
|
137
408
|
gateway: gatewayClient,
|
|
409
|
+
gatewayConfig: config.gateway,
|
|
410
|
+
paymentEnabled: config.payment.enabled,
|
|
411
|
+
gatewayUrl: config.gateway.url,
|
|
412
|
+
lineageGateway,
|
|
413
|
+
derivativeCompute,
|
|
414
|
+
config,
|
|
138
415
|
accessLogWriter,
|
|
139
416
|
accessLogReader,
|
|
417
|
+
readFulfillmentReporter: options?.readFulfillmentReporter,
|
|
418
|
+
dataStorage,
|
|
419
|
+
scopeDeletions,
|
|
420
|
+
cloudMode,
|
|
140
421
|
devToken,
|
|
422
|
+
ownerSignature: masterKeySignature,
|
|
423
|
+
ownerPrivateKey,
|
|
424
|
+
accessToken,
|
|
425
|
+
tokenStore,
|
|
141
426
|
configPath,
|
|
142
427
|
syncManager,
|
|
143
428
|
serverSigner,
|
|
144
429
|
getTunnelStatus: () => tunnelManager?.getStatus() ?? null,
|
|
430
|
+
mcpOAuthApprovalUrl: options?.mcpOAuthApprovalUrl,
|
|
431
|
+
onServerRegistered: (serverId) => notifyServerRegistered(serverId),
|
|
432
|
+
profile: options?.profile,
|
|
433
|
+
jobWorker,
|
|
145
434
|
});
|
|
146
435
|
const cleanup = async () => {
|
|
436
|
+
backgroundClosed = true;
|
|
437
|
+
clearRegistrationPoll();
|
|
438
|
+
derivativeScheduler.stop();
|
|
147
439
|
if (tunnelManager) {
|
|
148
440
|
await tunnelManager.stop();
|
|
149
441
|
}
|
|
@@ -165,8 +457,20 @@ export async function createServer(config, options) {
|
|
|
165
457
|
syncManager,
|
|
166
458
|
tunnelManager,
|
|
167
459
|
tunnelUrl: undefined,
|
|
460
|
+
isServerRegistered: () => Boolean(identity?.serverId),
|
|
461
|
+
localApprovalPort,
|
|
462
|
+
getLocalApprovalOrigin: () => effectiveLocalApprovalOrigin,
|
|
463
|
+
setLocalApprovalOrigin: (origin) => {
|
|
464
|
+
effectiveLocalApprovalOrigin = origin;
|
|
465
|
+
},
|
|
168
466
|
devToken,
|
|
169
467
|
startBackgroundServices: async () => {
|
|
468
|
+
// Replay questions a previous run left pending or stale: nothing
|
|
469
|
+
// else reschedules them after a restart (registration and source
|
|
470
|
+
// changes only cover new activity).
|
|
471
|
+
if (!isEnclave) {
|
|
472
|
+
derivativeScheduler.start();
|
|
473
|
+
}
|
|
170
474
|
// --- Gateway registration check (slow: HTTP call) ---
|
|
171
475
|
if (serverAccount && identity) {
|
|
172
476
|
try {
|
|
@@ -187,33 +491,124 @@ export async function createServer(config, options) {
|
|
|
187
491
|
}
|
|
188
492
|
}
|
|
189
493
|
// --- Tunnel setup (slow: subprocess wait) ---
|
|
190
|
-
if (
|
|
494
|
+
if (!isEnclave &&
|
|
495
|
+
config.tunnel.enabled &&
|
|
191
496
|
serverOwner &&
|
|
192
497
|
serverAccount &&
|
|
193
498
|
frpcBinaryPath) {
|
|
194
499
|
tunnelManager = new TunnelManager(storageRoot);
|
|
195
500
|
context.tunnelManager = tunnelManager;
|
|
196
501
|
const runId = randomUUID();
|
|
502
|
+
let tunnelConnectStarted = false;
|
|
503
|
+
const connectTunnel = async () => {
|
|
504
|
+
const manager = tunnelManager;
|
|
505
|
+
if (!manager || tunnelConnectStarted || backgroundClosed) {
|
|
506
|
+
return;
|
|
507
|
+
}
|
|
508
|
+
tunnelConnectStarted = true;
|
|
509
|
+
try {
|
|
510
|
+
const url = await manager.connect();
|
|
511
|
+
const tunnelStatus = manager.getStatus();
|
|
512
|
+
if (tunnelStatus.status === "connected") {
|
|
513
|
+
logger.info({ tunnelUrl: url }, "Tunnel established");
|
|
514
|
+
}
|
|
515
|
+
else {
|
|
516
|
+
logger.warn({ tunnelUrl: url, warning: tunnelStatus.warning }, "Tunnel URL reserved; waiting for tunnel connection");
|
|
517
|
+
}
|
|
518
|
+
}
|
|
519
|
+
catch (err) {
|
|
520
|
+
logger.warn({ err }, "Tunnel failed to connect - server running in local-only mode");
|
|
521
|
+
tunnelManager = undefined;
|
|
522
|
+
context.tunnelManager = undefined;
|
|
523
|
+
// Local-only mode must keep the local audience/URL: leaving the
|
|
524
|
+
// public URL as effectiveOrigin would reject local Web3Signed
|
|
525
|
+
// callers against an origin that will never route.
|
|
526
|
+
effectiveOrigin = config.server.origin;
|
|
527
|
+
context.tunnelUrl = undefined;
|
|
528
|
+
}
|
|
529
|
+
};
|
|
530
|
+
// Poll fallback for registrations the route callback can't see
|
|
531
|
+
// (e.g. desktop registers with the gateway directly and the sidecar
|
|
532
|
+
// only observes it via a gateway lookup).
|
|
533
|
+
const scheduleRegistrationPoll = () => {
|
|
534
|
+
const pollStartedAt = Date.now();
|
|
535
|
+
const poll = async () => {
|
|
536
|
+
registrationPollTimer = null;
|
|
537
|
+
if (backgroundClosed) {
|
|
538
|
+
return;
|
|
539
|
+
}
|
|
540
|
+
try {
|
|
541
|
+
const serverInfo = await gatewayClient.getServer(serverAccount.address);
|
|
542
|
+
if (serverInfo?.id) {
|
|
543
|
+
if (identity) {
|
|
544
|
+
identity.serverId = serverInfo.id;
|
|
545
|
+
}
|
|
546
|
+
logger.info("Server registration detected — starting tunnel");
|
|
547
|
+
await connectTunnel();
|
|
548
|
+
return;
|
|
549
|
+
}
|
|
550
|
+
}
|
|
551
|
+
catch {
|
|
552
|
+
// Gateway unreachable — keep polling.
|
|
553
|
+
}
|
|
554
|
+
if (backgroundClosed) {
|
|
555
|
+
return;
|
|
556
|
+
}
|
|
557
|
+
const interval = Date.now() - pollStartedAt < 2 * 60_000 ? 5_000 : 30_000;
|
|
558
|
+
registrationPollTimer = setTimeout(() => void poll(), interval);
|
|
559
|
+
registrationPollTimer.unref?.();
|
|
560
|
+
};
|
|
561
|
+
registrationPollTimer = setTimeout(() => void poll(), 5_000);
|
|
562
|
+
registrationPollTimer.unref?.();
|
|
563
|
+
};
|
|
564
|
+
notifyServerRegistered = (serverId) => {
|
|
565
|
+
if (backgroundClosed) {
|
|
566
|
+
return;
|
|
567
|
+
}
|
|
568
|
+
if (serverId) {
|
|
569
|
+
if (identity) {
|
|
570
|
+
identity.serverId = serverId;
|
|
571
|
+
}
|
|
572
|
+
// Only a confirmed serverId stops the poll: on a null result the
|
|
573
|
+
// poll keeps running so identity/health converge once the
|
|
574
|
+
// gateway lookup sees the registration.
|
|
575
|
+
clearRegistrationPoll();
|
|
576
|
+
}
|
|
577
|
+
void connectTunnel();
|
|
578
|
+
};
|
|
197
579
|
try {
|
|
198
|
-
|
|
580
|
+
// Reserve the public URL without dialing the relay: the URL is
|
|
581
|
+
// what registration needs, and dialing before the registration
|
|
582
|
+
// lands poisons the relay's delegation cache (BUI-611).
|
|
583
|
+
const url = tunnelManager.reserve({
|
|
199
584
|
walletAddress: serverAccount.address,
|
|
200
585
|
ownerAddress: serverOwner,
|
|
201
586
|
serverKeypair: serverAccount,
|
|
587
|
+
tunnelSigner: ownerTunnelSigner
|
|
588
|
+
? {
|
|
589
|
+
signMessage: (message) => ownerTunnelSigner.signMessage({ message }),
|
|
590
|
+
}
|
|
591
|
+
: undefined,
|
|
202
592
|
runId,
|
|
203
593
|
serverAddr: config.tunnel.serverAddr,
|
|
204
594
|
serverPort: config.tunnel.serverPort,
|
|
205
595
|
localPort: config.server.port,
|
|
206
596
|
}, frpcBinaryPath);
|
|
207
|
-
logger.info({ tunnelUrl: url }, "Tunnel established");
|
|
208
597
|
context.tunnelUrl = url;
|
|
598
|
+
// The public URL is the audience clients sign against, so adopt it
|
|
599
|
+
// as the effective origin as soon as it's known (matches the
|
|
600
|
+
// unity-surfaces #583 sidecar patch).
|
|
209
601
|
effectiveOrigin = url;
|
|
210
|
-
if (
|
|
211
|
-
|
|
212
|
-
|
|
602
|
+
if (identity?.serverId) {
|
|
603
|
+
await connectTunnel();
|
|
604
|
+
}
|
|
605
|
+
else {
|
|
606
|
+
logger.info({ tunnelUrl: url }, "Tunnel start deferred until server registration — register to enable routing");
|
|
607
|
+
scheduleRegistrationPoll();
|
|
213
608
|
}
|
|
214
609
|
}
|
|
215
610
|
catch (err) {
|
|
216
|
-
logger.warn({ err }, "Tunnel failed to
|
|
611
|
+
logger.warn({ err }, "Tunnel failed to start - server running in local-only mode");
|
|
217
612
|
tunnelManager = undefined;
|
|
218
613
|
context.tunnelManager = undefined;
|
|
219
614
|
}
|