infinite-canvas-local-proxy 0.2.0 → 0.2.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +39 -3
- package/index.js +7 -16
- package/memory-task-store.js +132 -0
- package/package.json +4 -2
- package/script-runner.js +184 -18
- package/sqlite-backend.js +102 -0
- package/task-store.js +9 -5
package/README.md
CHANGED
|
@@ -56,8 +56,8 @@ http://127.0.0.1:23210/https://api.openai.com/v1/models
|
|
|
56
56
|
- 默认数据目录为用户主目录下的 `.canvas-proxy`(Windows 为 `%USERPROFILE%\.canvas-proxy`),包括 `tasks.sqlite`、SQLite 日志文件和 `results/` 响应文件。备份时需停止代理后复制整个目录。
|
|
57
57
|
- API Key、请求头和请求体仅在代理内存中用于上游请求;任务请求体、脚本上下文及 API Key 不写入数据库或任务日志。若上游响应或脚本错误中发现已识别的凭证原文,该结果不会保存或会在错误中脱敏。
|
|
58
58
|
- 图片/视频脚本请求支持 JSON、文本、URL 编码和 multipart/form-data;multipart 文件在本地任务创建请求中临时 Base64 编码,代理端重建为 multipart 后发送。请求体总大小上限 80 MiB;独立二进制请求体不支持。
|
|
59
|
-
-
|
|
60
|
-
- 可通过环境变量 `CANVAS_PROXY_DATA_DIR`、`
|
|
59
|
+
- 任务数量、并发数和单个生成脚本的请求次数均不设上限;任务提交后会立即启动。生成任务仍受单任务 30 分钟等待时限、单次上游请求 10 分钟超时、80 MiB 请求体和 100 MiB 响应大小限制。成功/失败记录保留 24 小时。
|
|
60
|
+
- 可通过环境变量 `CANVAS_PROXY_DATA_DIR`、`CANVAS_PROXY_TASK_REQUEST_LIMIT_BYTES`、`CANVAS_PROXY_TASK_RESPONSE_LIMIT_BYTES`、`CANVAS_PROXY_TASK_TIMEOUT_MS` 和 `CANVAS_PROXY_TASK_RETENTION_MS` 调整目录与限制。
|
|
61
61
|
- 代理进程重启时,原来排队或运行中的任务标记为“结果未知”,不会自动再次提交上游请求,以避免重复计费。已完成结果仍可从数据库记录和响应文件恢复,直至过期。
|
|
62
62
|
- 代理只报告 queued/running/polling/完成/失败等本地状态,不提供或伪造供应商生成百分比。代理必须保持运行,才能在浏览器关闭后继续执行代理持有的请求或自定义脚本轮询。
|
|
63
63
|
|
|
@@ -69,4 +69,40 @@ http://127.0.0.1:23210/https://api.openai.com/v1/models
|
|
|
69
69
|
|
|
70
70
|
## 环境要求
|
|
71
71
|
|
|
72
|
-
Node.js >= 18
|
|
72
|
+
Node.js >= 18。不需要 Python、Visual Studio 编译工具,也不需要任何原生模块通过编译——安装阶段不会触发 `node-gyp`。
|
|
73
|
+
|
|
74
|
+
### 任务存储的三种后端
|
|
75
|
+
|
|
76
|
+
代理会按顺序自动挑选,不需要手动配置:
|
|
77
|
+
|
|
78
|
+
| 后端 | 条件 | 任务是否持久化 |
|
|
79
|
+
|---|---|---|
|
|
80
|
+
| `better-sqlite3` | 安装成功且与当前 Node 的 ABI 匹配 | 是 |
|
|
81
|
+
| `node:sqlite` | Node 22.5 及以上(内置) | 是 |
|
|
82
|
+
| `memory` | 上面两者都不可用 | 否,重启后清空 |
|
|
83
|
+
|
|
84
|
+
启动日志和根路径 `GET /` 会显示当前使用的后端,例如:
|
|
85
|
+
|
|
86
|
+
```json
|
|
87
|
+
{"taskStore":"node:sqlite","persistentTasks":true,"node":"22.22.2"}
|
|
88
|
+
```
|
|
89
|
+
|
|
90
|
+
`better-sqlite3` 是可选依赖。某些 Node 版本(如 24)没有对应的预编译产物,安装时会尝试本地编译并失败,但 **npm 会跳过它并继续**,代理随后自动改用内置的 `node:sqlite`,功能不受影响。
|
|
91
|
+
|
|
92
|
+
Node 22.5 及以上首次加载内置 SQLite 时会打印一行 `ExperimentalWarning: SQLite is an experimental feature`,这是 Node 自身的提示,不影响使用。
|
|
93
|
+
|
|
94
|
+
### 安装报错的排查
|
|
95
|
+
|
|
96
|
+
如果看到 `prebuild-install warn install No prebuilt binaries found` 并跟着一长串 `gyp ERR! find Python`,说明当前 Node 版本没有 `better-sqlite3` 的预编译产物,于是退回到本地编译,而机器上又没装 Python。这种情况在新版本 Node 上很常见。
|
|
97
|
+
|
|
98
|
+
0.2.1 起这个错误不会再中断安装。如果仍想彻底消除,任选其一:
|
|
99
|
+
|
|
100
|
+
- 升级到 Node 22 LTS 或更高,走内置 `node:sqlite`,完全不需要编译
|
|
101
|
+
- 或者安装 Python 3 与 Visual Studio Build Tools,让 `better-sqlite3` 能编译
|
|
102
|
+
|
|
103
|
+
清除安装残留:
|
|
104
|
+
|
|
105
|
+
```bash
|
|
106
|
+
rmdir /s /q "%LocalAppData%\npm-cache\_npx" # Windows
|
|
107
|
+
npm cache clean --force
|
|
108
|
+
```
|
package/index.js
CHANGED
|
@@ -28,8 +28,6 @@ const TASK_TIMEOUT_MS = 30 * 60 * 1000;
|
|
|
28
28
|
const TASK_RETENTION_MS = 24 * 60 * 60 * 1000;
|
|
29
29
|
const TASK_REQUEST_LIMIT = 80 * 1024 * 1024;
|
|
30
30
|
const TASK_RESPONSE_LIMIT = 100 * 1024 * 1024;
|
|
31
|
-
const TASK_CONCURRENCY = 2;
|
|
32
|
-
const TASK_OUTSTANDING_LIMIT = 4;
|
|
33
31
|
|
|
34
32
|
function positiveInteger(value, fallback) {
|
|
35
33
|
const parsed = Number(value);
|
|
@@ -226,26 +224,26 @@ async function forward(req, res, target) {
|
|
|
226
224
|
export function createProxyServer(options = {}) {
|
|
227
225
|
const dataDir = options.dataDir || process.env.CANVAS_PROXY_DATA_DIR || path.join(os.homedir(), ".canvas-proxy");
|
|
228
226
|
const store = createTaskStore(dataDir);
|
|
227
|
+
if (!store.persistent) {
|
|
228
|
+
console.warn(`任务存储: 内存模式(未找到可用的 SQLite),重启后任务记录不会保留。升级到 Node 22.5+ 可自动启用内置 SQLite。`);
|
|
229
|
+
console.warn(`当前 Node ${process.versions.node}。`);
|
|
230
|
+
}
|
|
229
231
|
const now = Date.now();
|
|
230
232
|
store.recover(now);
|
|
231
233
|
const taskTimeoutMs = positiveInteger(options.taskTimeoutMs ?? process.env.CANVAS_PROXY_TASK_TIMEOUT_MS, TASK_TIMEOUT_MS);
|
|
232
234
|
const taskRetentionMs = positiveInteger(options.taskRetentionMs ?? process.env.CANVAS_PROXY_TASK_RETENTION_MS, TASK_RETENTION_MS);
|
|
233
235
|
const taskRequestLimit = positiveInteger(options.taskRequestLimit ?? process.env.CANVAS_PROXY_TASK_REQUEST_LIMIT_BYTES, TASK_REQUEST_LIMIT);
|
|
234
236
|
const taskResponseLimit = positiveInteger(options.taskResponseLimit ?? process.env.CANVAS_PROXY_TASK_RESPONSE_LIMIT_BYTES, TASK_RESPONSE_LIMIT);
|
|
235
|
-
const taskConcurrency = positiveInteger(options.taskConcurrency ?? process.env.CANVAS_PROXY_TASK_CONCURRENCY, TASK_CONCURRENCY);
|
|
236
|
-
const taskOutstandingLimit = positiveInteger(options.taskOutstandingLimit ?? process.env.CANVAS_PROXY_TASK_OUTSTANDING_LIMIT, TASK_OUTSTANDING_LIMIT);
|
|
237
237
|
const queuedRequests = new Map();
|
|
238
|
-
let runningTasks = 0;
|
|
239
238
|
|
|
240
239
|
const sendTaskError = (res, status, error) => sendJson(res, status, { error });
|
|
241
240
|
|
|
242
241
|
const startQueuedTasks = () => {
|
|
243
|
-
while (
|
|
242
|
+
while (true) {
|
|
244
243
|
const next = queuedRequests.entries().next();
|
|
245
244
|
if (next.done) return;
|
|
246
245
|
const [taskId, taskRequest] = next.value;
|
|
247
246
|
queuedRequests.delete(taskId);
|
|
248
|
-
runningTasks += 1;
|
|
249
247
|
const execute = taskRequest.runner === "script" ? executeScriptTask : taskRequest.runner === "video" ? executeVideoTask : executeTask;
|
|
250
248
|
void execute(taskId, taskRequest)
|
|
251
249
|
.catch(() => {
|
|
@@ -256,10 +254,7 @@ export function createProxyServer(options = {}) {
|
|
|
256
254
|
}
|
|
257
255
|
console.error("Persistent task worker failed; it will not be retried automatically.");
|
|
258
256
|
})
|
|
259
|
-
.finally(
|
|
260
|
-
runningTasks -= 1;
|
|
261
|
-
startQueuedTasks();
|
|
262
|
-
});
|
|
257
|
+
.finally(startQueuedTasks);
|
|
263
258
|
}
|
|
264
259
|
};
|
|
265
260
|
|
|
@@ -709,10 +704,6 @@ export function createProxyServer(options = {}) {
|
|
|
709
704
|
return;
|
|
710
705
|
}
|
|
711
706
|
}
|
|
712
|
-
if (runningTasks + queuedRequests.size >= taskOutstandingLimit) {
|
|
713
|
-
sendTaskError(res, 429, "The local image task queue is full.");
|
|
714
|
-
return;
|
|
715
|
-
}
|
|
716
707
|
const taskId = randomBytes(32).toString("base64url");
|
|
717
708
|
const createdAt = Date.now();
|
|
718
709
|
const created = store.createOrGet({
|
|
@@ -774,7 +765,7 @@ export function createProxyServer(options = {}) {
|
|
|
774
765
|
|
|
775
766
|
const target = readTarget(req.url || "/");
|
|
776
767
|
if (!target) {
|
|
777
|
-
sendJson(res, 200, { app: "infinite-canvas", proxy: pkg.name, version: pkg.version, taskApiVersion: 1, usage: "/<full-target-url>" });
|
|
768
|
+
sendJson(res, 200, { app: "infinite-canvas", proxy: pkg.name, version: pkg.version, taskApiVersion: 1, taskStore: store.backend, persistentTasks: store.persistent, node: process.versions.node, usage: "/<full-target-url>" });
|
|
778
769
|
return;
|
|
779
770
|
}
|
|
780
771
|
const startedAt = Date.now();
|
|
@@ -0,0 +1,132 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* 内存版任务存储。只在 better-sqlite3 与 node:sqlite 都不可用时启用,
|
|
3
|
+
* 目的是让代理仍然能启动、转发照常工作,代价是重启后任务记录不再保留。
|
|
4
|
+
* 接口与基于 SQLite 的实现保持一致,主程序无需区分。
|
|
5
|
+
*/
|
|
6
|
+
export function createMemoryTaskStore(resultDir) {
|
|
7
|
+
const tasks = new Map();
|
|
8
|
+
|
|
9
|
+
function toTask(row) {
|
|
10
|
+
if (!row) return null;
|
|
11
|
+
let metadata;
|
|
12
|
+
if (row.metadata_json) {
|
|
13
|
+
metadata = JSON.parse(row.metadata_json);
|
|
14
|
+
}
|
|
15
|
+
return {
|
|
16
|
+
id: row.id,
|
|
17
|
+
status: row.status,
|
|
18
|
+
createdAt: row.created_at,
|
|
19
|
+
updatedAt: row.updated_at,
|
|
20
|
+
expiresAt: row.expires_at,
|
|
21
|
+
responseStatus: row.response_status,
|
|
22
|
+
responseContentType: row.response_content_type,
|
|
23
|
+
responseFile: row.response_file,
|
|
24
|
+
error: row.error,
|
|
25
|
+
...(row.client_request_id ? { clientRequestId: row.client_request_id } : {}),
|
|
26
|
+
...(row.kind !== "request" ? { kind: row.kind } : {}),
|
|
27
|
+
...(row.source ? { source: row.source } : {}),
|
|
28
|
+
...(metadata ? { metadata } : {}),
|
|
29
|
+
};
|
|
30
|
+
}
|
|
31
|
+
|
|
32
|
+
function insert(task) {
|
|
33
|
+
const row = {
|
|
34
|
+
id: task.id,
|
|
35
|
+
status: "queued",
|
|
36
|
+
created_at: task.now,
|
|
37
|
+
updated_at: task.now,
|
|
38
|
+
expires_at: task.expiresAt,
|
|
39
|
+
response_status: null,
|
|
40
|
+
response_content_type: null,
|
|
41
|
+
response_file: null,
|
|
42
|
+
error: null,
|
|
43
|
+
client_request_id: task.clientRequestId || null,
|
|
44
|
+
kind: task.kind || "request",
|
|
45
|
+
source: task.source || null,
|
|
46
|
+
metadata_json: task.metadata ? JSON.stringify(task.metadata) : null,
|
|
47
|
+
};
|
|
48
|
+
tasks.set(row.id, row);
|
|
49
|
+
return row;
|
|
50
|
+
}
|
|
51
|
+
|
|
52
|
+
function findByClientRequestId(clientRequestId) {
|
|
53
|
+
if (!clientRequestId) return null;
|
|
54
|
+
for (const row of tasks.values()) {
|
|
55
|
+
if (row.client_request_id === clientRequestId) return row;
|
|
56
|
+
}
|
|
57
|
+
return null;
|
|
58
|
+
}
|
|
59
|
+
|
|
60
|
+
return {
|
|
61
|
+
resultDir,
|
|
62
|
+
backend: "memory",
|
|
63
|
+
persistent: false,
|
|
64
|
+
create(id, now, expiresAt) {
|
|
65
|
+
insert({ id, now, expiresAt, clientRequestId: null, kind: "request", source: null, metadata: null });
|
|
66
|
+
},
|
|
67
|
+
createOrGet(task) {
|
|
68
|
+
const resolved = { kind: "request", source: null, metadata: null, ...task };
|
|
69
|
+
if (resolved.clientRequestId) {
|
|
70
|
+
const existing = findByClientRequestId(resolved.clientRequestId);
|
|
71
|
+
if (existing) return { task: toTask(existing), created: false };
|
|
72
|
+
}
|
|
73
|
+
const row = insert(resolved);
|
|
74
|
+
return { task: toTask(row), created: true };
|
|
75
|
+
},
|
|
76
|
+
setStatus(id, status, now, error = null) {
|
|
77
|
+
const row = tasks.get(id);
|
|
78
|
+
if (!row) return;
|
|
79
|
+
row.status = status;
|
|
80
|
+
row.updated_at = now;
|
|
81
|
+
row.error = error;
|
|
82
|
+
},
|
|
83
|
+
setSucceeded(id, now, status, contentType, file) {
|
|
84
|
+
const row = tasks.get(id);
|
|
85
|
+
if (!row) return;
|
|
86
|
+
row.status = "succeeded";
|
|
87
|
+
row.updated_at = now;
|
|
88
|
+
row.response_status = status;
|
|
89
|
+
row.response_content_type = contentType;
|
|
90
|
+
row.response_file = file;
|
|
91
|
+
row.error = null;
|
|
92
|
+
},
|
|
93
|
+
get(id) {
|
|
94
|
+
return toTask(tasks.get(id));
|
|
95
|
+
},
|
|
96
|
+
getByClientRequestId(clientRequestId) {
|
|
97
|
+
return toTask(findByClientRequestId(clientRequestId));
|
|
98
|
+
},
|
|
99
|
+
listActive(now, limit = 100) {
|
|
100
|
+
const active = ["queued", "running", "submitting", "polling"];
|
|
101
|
+
return [...tasks.values()]
|
|
102
|
+
.filter((row) => row.expires_at > now && active.includes(row.status))
|
|
103
|
+
.sort((a, b) => b.created_at - a.created_at)
|
|
104
|
+
.slice(0, limit)
|
|
105
|
+
.map(toTask);
|
|
106
|
+
},
|
|
107
|
+
recover(now) {
|
|
108
|
+
const active = ["queued", "running", "submitting", "polling"];
|
|
109
|
+
let changes = 0;
|
|
110
|
+
for (const row of tasks.values()) {
|
|
111
|
+
if (!active.includes(row.status)) continue;
|
|
112
|
+
row.status = "unknown";
|
|
113
|
+
row.updated_at = now;
|
|
114
|
+
row.error = "Proxy restarted while the upstream request was running; its outcome is unknown.";
|
|
115
|
+
changes += 1;
|
|
116
|
+
}
|
|
117
|
+
return changes;
|
|
118
|
+
},
|
|
119
|
+
removeExpired(now) {
|
|
120
|
+
const finished = ["succeeded", "failed", "unknown"];
|
|
121
|
+
return [...tasks.values()]
|
|
122
|
+
.filter((row) => row.expires_at <= now && finished.includes(row.status))
|
|
123
|
+
.map((row) => ({ id: row.id, response_file: row.response_file }));
|
|
124
|
+
},
|
|
125
|
+
remove(id) {
|
|
126
|
+
tasks.delete(id);
|
|
127
|
+
},
|
|
128
|
+
close() {
|
|
129
|
+
tasks.clear();
|
|
130
|
+
},
|
|
131
|
+
};
|
|
132
|
+
}
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "infinite-canvas-local-proxy",
|
|
3
|
-
"version": "0.2.
|
|
3
|
+
"version": "0.2.2",
|
|
4
4
|
"description": "Minimal local forwarding proxy that lets the Infinite Canvas web app call AI provider APIs without CORS restrictions.",
|
|
5
5
|
"license": "MIT",
|
|
6
6
|
"type": "module",
|
|
@@ -11,6 +11,8 @@
|
|
|
11
11
|
"files": [
|
|
12
12
|
"index.js",
|
|
13
13
|
"task-store.js",
|
|
14
|
+
"sqlite-backend.js",
|
|
15
|
+
"memory-task-store.js",
|
|
14
16
|
"script-runner.js",
|
|
15
17
|
"README.md"
|
|
16
18
|
],
|
|
@@ -21,7 +23,7 @@
|
|
|
21
23
|
"engines": {
|
|
22
24
|
"node": ">=18"
|
|
23
25
|
},
|
|
24
|
-
"
|
|
26
|
+
"optionalDependencies": {
|
|
25
27
|
"better-sqlite3": "^11.8.1"
|
|
26
28
|
},
|
|
27
29
|
"publishConfig": {
|
package/script-runner.js
CHANGED
|
@@ -4,9 +4,27 @@ import vm from "node:vm";
|
|
|
4
4
|
|
|
5
5
|
const SCRIPT_TIMEOUT_MS = 30 * 60 * 1000;
|
|
6
6
|
const REQUEST_TIMEOUT_MS = 10 * 60 * 1000;
|
|
7
|
-
const MAX_SCRIPT_REQUESTS = 32;
|
|
8
7
|
const mediaUploadCache = new Map();
|
|
9
8
|
const MEDIA_UPLOAD_CACHE_LIMIT = 500;
|
|
9
|
+
const abortPromises = new WeakMap();
|
|
10
|
+
|
|
11
|
+
/** 每个任务信号只注册一次 abort 监听:轮询会反复执行脚本,逐次注册会累积成内存泄漏。 */
|
|
12
|
+
function abortPromiseFor(signal) {
|
|
13
|
+
if (!signal) return new Promise(() => {});
|
|
14
|
+
const cached = abortPromises.get(signal);
|
|
15
|
+
if (cached) return cached;
|
|
16
|
+
const promise = new Promise((_, reject) => {
|
|
17
|
+
const fail = () => reject(new DOMException("Aborted", "AbortError"));
|
|
18
|
+
if (signal.aborted) {
|
|
19
|
+
fail();
|
|
20
|
+
return;
|
|
21
|
+
}
|
|
22
|
+
signal.addEventListener("abort", fail, { once: true });
|
|
23
|
+
});
|
|
24
|
+
promise.catch(() => {});
|
|
25
|
+
abortPromises.set(signal, promise);
|
|
26
|
+
return promise;
|
|
27
|
+
}
|
|
10
28
|
|
|
11
29
|
class ScriptFileReader {
|
|
12
30
|
result = null;
|
|
@@ -75,6 +93,100 @@ function makeFile(value, fallbackName) {
|
|
|
75
93
|
});
|
|
76
94
|
}
|
|
77
95
|
|
|
96
|
+
function sniffMime(bytes) {
|
|
97
|
+
const ascii = (start, end) => bytes.toString("ascii", start, end);
|
|
98
|
+
if (bytes.subarray(0, 8).equals(Buffer.from([0x89, 0x50, 0x4e, 0x47, 0x0d, 0x0a, 0x1a, 0x0a]))) return "image/png";
|
|
99
|
+
if (bytes[0] === 0xff && bytes[1] === 0xd8 && bytes[2] === 0xff) return "image/jpeg";
|
|
100
|
+
if (ascii(0, 4) === "RIFF" && ascii(8, 12) === "WEBP") return "image/webp";
|
|
101
|
+
if (ascii(0, 4) === "GIF8") return "image/gif";
|
|
102
|
+
if (ascii(0, 4) === "RIFF" && ascii(8, 12) === "WAVE") return "audio/wav";
|
|
103
|
+
if (ascii(0, 3) === "ID3" || (bytes[0] === 0xff && (bytes[1] & 0xe0) === 0xe0 && (bytes[1] & 0x06) !== 0)) return "audio/mpeg";
|
|
104
|
+
if (bytes.subarray(0, 4).equals(Buffer.from([0x1a, 0x45, 0xdf, 0xa3]))) return "video/webm";
|
|
105
|
+
if (ascii(0, 4) === "OggS") return "audio/ogg";
|
|
106
|
+
if (ascii(4, 8) === "ftyp") {
|
|
107
|
+
const brands = [ascii(8, 12), ...Array.from({ length: Math.floor(Math.max(0, bytes.length - 16) / 4) }, (_, index) => ascii(16 + index * 4, 20 + index * 4))];
|
|
108
|
+
if (brands.includes("qt ")) return "video/quicktime";
|
|
109
|
+
if (brands.some((brand) => ["M4A ", "M4B ", "M4P "].includes(brand))) return "audio/mp4";
|
|
110
|
+
if (brands.some((brand) => ["avif", "avis"].includes(brand))) return "image/avif";
|
|
111
|
+
if (brands.some((brand) => ["heic", "heix", "hevc", "hevx"].includes(brand))) return "image/heic";
|
|
112
|
+
if (brands.some((brand) => ["mif1", "msf1"].includes(brand))) return "image/heif";
|
|
113
|
+
return "video/mp4";
|
|
114
|
+
}
|
|
115
|
+
return null;
|
|
116
|
+
}
|
|
117
|
+
|
|
118
|
+
export function createScriptOssUploader(endpoint, namespace, scriptFetch, signal) {
|
|
119
|
+
const mimeExtensions = {
|
|
120
|
+
"image/avif": "avif",
|
|
121
|
+
"image/gif": "gif",
|
|
122
|
+
"image/heic": "heic",
|
|
123
|
+
"image/heif": "heif",
|
|
124
|
+
"image/jpeg": "jpg",
|
|
125
|
+
"image/png": "png",
|
|
126
|
+
"image/webp": "webp",
|
|
127
|
+
"video/mp4": "mp4",
|
|
128
|
+
"video/quicktime": "mov",
|
|
129
|
+
"video/webm": "webm",
|
|
130
|
+
"audio/mp4": "m4a",
|
|
131
|
+
"audio/mpeg": "mp3",
|
|
132
|
+
"audio/ogg": "ogg",
|
|
133
|
+
"audio/wav": "wav",
|
|
134
|
+
"audio/x-wav": "wav",
|
|
135
|
+
};
|
|
136
|
+
return async (file) => {
|
|
137
|
+
if (!(file instanceof Blob)) throw new Error("uploadToOss requires a Blob or File.");
|
|
138
|
+
if (typeof endpoint !== "string" || !endpoint.trim()) throw new Error("The application OSS presign endpoint was not provided to the local proxy.");
|
|
139
|
+
let presignEndpoint;
|
|
140
|
+
try {
|
|
141
|
+
presignEndpoint = new URL(endpoint);
|
|
142
|
+
} catch {
|
|
143
|
+
throw new Error("The application OSS presign endpoint must be an absolute URL.");
|
|
144
|
+
}
|
|
145
|
+
if (!["http:", "https:"].includes(presignEndpoint.protocol)) throw new Error("The application OSS presign endpoint must use HTTP or HTTPS.");
|
|
146
|
+
return cacheMediaUpload(file, `${namespace}:oss-mime-v2`, async () => {
|
|
147
|
+
const declaredType = file.type.split(";", 1)[0].trim().toLowerCase();
|
|
148
|
+
const contentType = declaredType && declaredType !== "application/octet-stream" ? declaredType : sniffMime(Buffer.from(await file.slice(0, 32).arrayBuffer()));
|
|
149
|
+
if (!contentType) throw new Error("Could not identify the reference media type from its content.");
|
|
150
|
+
const extension = mimeExtensions[contentType];
|
|
151
|
+
if (!extension) throw new Error(`OSS upload does not support MIME type "${contentType}".`);
|
|
152
|
+
const originalName = typeof file.name === "string" ? file.name : "reference";
|
|
153
|
+
const basename = originalName.split(/[\\/]/).at(-1)?.replace(/\.[^.]*$/, "") || "reference";
|
|
154
|
+
const presignResponse = await scriptFetch(endpoint, {
|
|
155
|
+
method: "POST",
|
|
156
|
+
headers: { "Content-Type": "application/json" },
|
|
157
|
+
body: JSON.stringify({ filename: `${basename}.${extension}`, contentType }),
|
|
158
|
+
signal,
|
|
159
|
+
});
|
|
160
|
+
const presignText = await presignResponse.text();
|
|
161
|
+
let presign;
|
|
162
|
+
try {
|
|
163
|
+
presign = presignText ? JSON.parse(presignText) : {};
|
|
164
|
+
} catch {
|
|
165
|
+
throw new Error(`OSS presign returned non-JSON (HTTP ${presignResponse.status}).`);
|
|
166
|
+
}
|
|
167
|
+
if (!presignResponse.ok) {
|
|
168
|
+
const detail = typeof presign.error === "string" ? presign.error : typeof presign.message === "string" ? presign.message : "";
|
|
169
|
+
throw new Error(detail || `Could not get OSS upload URL (HTTP ${presignResponse.status}).`);
|
|
170
|
+
}
|
|
171
|
+
if (typeof presign.uploadUrl !== "string" || typeof presign.publicUrl !== "string") {
|
|
172
|
+
throw new Error("OSS presign response must include uploadUrl and publicUrl.");
|
|
173
|
+
}
|
|
174
|
+
if (new URL(presign.uploadUrl).protocol !== "https:" || new URL(presign.publicUrl).protocol !== "https:") {
|
|
175
|
+
throw new Error("OSS upload and public media URLs must use HTTPS.");
|
|
176
|
+
}
|
|
177
|
+
const uploadContentType = typeof presign.contentType === "string" && presign.contentType ? presign.contentType : contentType;
|
|
178
|
+
const uploaded = await scriptFetch(presign.uploadUrl, {
|
|
179
|
+
method: "PUT",
|
|
180
|
+
headers: { "Content-Type": uploadContentType },
|
|
181
|
+
body: file,
|
|
182
|
+
signal,
|
|
183
|
+
});
|
|
184
|
+
if (!uploaded.ok) throw new Error(`OSS upload failed: HTTP ${uploaded.status}.`);
|
|
185
|
+
return presign.publicUrl;
|
|
186
|
+
});
|
|
187
|
+
};
|
|
188
|
+
}
|
|
189
|
+
|
|
78
190
|
function bodySize(data) {
|
|
79
191
|
if (data === undefined || data === null) return 0;
|
|
80
192
|
if (typeof data === "string") return Buffer.byteLength(data);
|
|
@@ -90,6 +202,29 @@ function bodySize(data) {
|
|
|
90
202
|
return Buffer.byteLength(JSON.stringify(data));
|
|
91
203
|
}
|
|
92
204
|
|
|
205
|
+
function composeAbortSignal(signals, timeoutMs) {
|
|
206
|
+
const controller = new AbortController();
|
|
207
|
+
const listeners = [];
|
|
208
|
+
for (const signal of signals) {
|
|
209
|
+
if (!signal) continue;
|
|
210
|
+
if (signal.aborted) {
|
|
211
|
+
controller.abort(signal.reason);
|
|
212
|
+
break;
|
|
213
|
+
}
|
|
214
|
+
const abort = () => controller.abort(signal.reason);
|
|
215
|
+
signal.addEventListener("abort", abort, { once: true });
|
|
216
|
+
listeners.push([signal, abort]);
|
|
217
|
+
}
|
|
218
|
+
const timeout = setTimeout(() => controller.abort(new DOMException("The operation timed out.", "TimeoutError")), timeoutMs);
|
|
219
|
+
return {
|
|
220
|
+
signal: controller.signal,
|
|
221
|
+
dispose() {
|
|
222
|
+
clearTimeout(timeout);
|
|
223
|
+
for (const [signal, abort] of listeners) signal.removeEventListener("abort", abort);
|
|
224
|
+
},
|
|
225
|
+
};
|
|
226
|
+
}
|
|
227
|
+
|
|
93
228
|
async function responseData(response, responseType, limitBytes) {
|
|
94
229
|
const contentType = response.headers.get("content-type") || "";
|
|
95
230
|
const bytes = Buffer.from(await response.arrayBuffer());
|
|
@@ -108,11 +243,6 @@ async function responseData(response, responseType, limitBytes) {
|
|
|
108
243
|
}
|
|
109
244
|
|
|
110
245
|
function createRequest({ signal, requestLimitBytes, responseLimitBytes, credentials, baseUrl }) {
|
|
111
|
-
let requestCount = 0;
|
|
112
|
-
const countRequest = () => {
|
|
113
|
-
requestCount += 1;
|
|
114
|
-
if (requestCount > MAX_SCRIPT_REQUESTS) throw new Error("A generation script exceeded the request count limit.");
|
|
115
|
-
};
|
|
116
246
|
const recordCredentials = (target, headers) => {
|
|
117
247
|
for (const [name, value] of headers.entries()) {
|
|
118
248
|
if (/authorization|api.?key/i.test(name)) credentials.add(value.replace(/^bearer\s+/i, ""));
|
|
@@ -124,7 +254,6 @@ function createRequest({ signal, requestLimitBytes, responseLimitBytes, credenti
|
|
|
124
254
|
}
|
|
125
255
|
};
|
|
126
256
|
const request = async (options) => {
|
|
127
|
-
countRequest();
|
|
128
257
|
if (!isRecord(options) || typeof options.url !== "string") throw new Error("A generation script request needs a URL.");
|
|
129
258
|
const requestBase = String(baseUrl || "").trim();
|
|
130
259
|
const target = /^https?:\/\//i.test(options.url) ? new URL(options.url) : new URL(options.url, `${requestBase.replace(/\/+$/, "")}/`);
|
|
@@ -180,7 +309,6 @@ function createRequest({ signal, requestLimitBytes, responseLimitBytes, credenti
|
|
|
180
309
|
}
|
|
181
310
|
};
|
|
182
311
|
const scriptFetch = async (input, init = {}) => {
|
|
183
|
-
countRequest();
|
|
184
312
|
const requestInput = input instanceof Request ? input : null;
|
|
185
313
|
const target = new URL(requestInput ? requestInput.url : String(input));
|
|
186
314
|
if (!["http:", "https:", "data:"].includes(target.protocol)) throw new Error("Generation scripts may only fetch HTTP, HTTPS or data URLs.");
|
|
@@ -190,17 +318,53 @@ function createRequest({ signal, requestLimitBytes, responseLimitBytes, credenti
|
|
|
190
318
|
if (init.body instanceof FormData && /^multipart\/form-data\s*$/i.test(headers.get("content-type") || "")) headers.delete("content-type");
|
|
191
319
|
recordCredentials(target, headers);
|
|
192
320
|
if (bodySize(init.body) > requestLimitBytes) throw new Error("Generation script request exceeds the configured size limit.");
|
|
193
|
-
const
|
|
194
|
-
|
|
195
|
-
|
|
321
|
+
const composed = composeAbortSignal([signal, init.signal || requestInput?.signal], REQUEST_TIMEOUT_MS);
|
|
322
|
+
let response;
|
|
323
|
+
try {
|
|
324
|
+
response = await fetch(input, { ...init, headers, signal: composed.signal });
|
|
325
|
+
} catch (error) {
|
|
326
|
+
composed.dispose();
|
|
327
|
+
throw error;
|
|
328
|
+
}
|
|
329
|
+
if (!response.body) {
|
|
330
|
+
composed.dispose();
|
|
331
|
+
return response;
|
|
332
|
+
}
|
|
333
|
+
const reader = response.body.getReader();
|
|
196
334
|
let received = 0;
|
|
197
|
-
const limited =
|
|
198
|
-
|
|
199
|
-
|
|
200
|
-
|
|
201
|
-
|
|
335
|
+
const limited = new ReadableStream({
|
|
336
|
+
async pull(controller) {
|
|
337
|
+
try {
|
|
338
|
+
const { done, value } = await reader.read();
|
|
339
|
+
if (done) {
|
|
340
|
+
controller.close();
|
|
341
|
+
composed.dispose();
|
|
342
|
+
return;
|
|
343
|
+
}
|
|
344
|
+
received += value.byteLength;
|
|
345
|
+
if (received > responseLimitBytes) {
|
|
346
|
+
try {
|
|
347
|
+
await reader.cancel();
|
|
348
|
+
} finally {
|
|
349
|
+
composed.dispose();
|
|
350
|
+
}
|
|
351
|
+
controller.error(new Error("Script response exceeds the configured size limit."));
|
|
352
|
+
return;
|
|
353
|
+
}
|
|
354
|
+
controller.enqueue(value);
|
|
355
|
+
} catch (error) {
|
|
356
|
+
controller.error(error);
|
|
357
|
+
composed.dispose();
|
|
358
|
+
}
|
|
202
359
|
},
|
|
203
|
-
|
|
360
|
+
async cancel(reason) {
|
|
361
|
+
try {
|
|
362
|
+
await reader.cancel(reason);
|
|
363
|
+
} finally {
|
|
364
|
+
composed.dispose();
|
|
365
|
+
}
|
|
366
|
+
},
|
|
367
|
+
});
|
|
204
368
|
const wrapped = new Response(limited, { status: response.status, statusText: response.statusText, headers: response.headers });
|
|
205
369
|
Object.defineProperties(wrapped, {
|
|
206
370
|
url: { value: response.url },
|
|
@@ -282,6 +446,7 @@ async function runScript(script, input, request, scriptFetch, signal, timeoutMs)
|
|
|
282
446
|
sleep,
|
|
283
447
|
signal,
|
|
284
448
|
cacheMediaUpload,
|
|
449
|
+
uploadToOss: createScriptOssUploader(input.ossPresignUrl, input.baseUrl, scriptFetch, signal),
|
|
285
450
|
console: Object.freeze({ log() {}, warn() {}, error() {} }),
|
|
286
451
|
});
|
|
287
452
|
const context = vm.createContext(sandbox, { codeGeneration: { strings: false, wasm: false } });
|
|
@@ -291,7 +456,7 @@ async function runScript(script, input, request, scriptFetch, signal, timeoutMs)
|
|
|
291
456
|
const timedOut = new Promise((_, reject) => {
|
|
292
457
|
timeout = setTimeout(() => reject(new Error("Generation script timed out.")), timeoutMs);
|
|
293
458
|
});
|
|
294
|
-
const aborted =
|
|
459
|
+
const aborted = abortPromiseFor(signal);
|
|
295
460
|
try {
|
|
296
461
|
return await Promise.race([result, timedOut, aborted]);
|
|
297
462
|
} finally {
|
|
@@ -344,6 +509,7 @@ export async function runGenerationScriptTask(task, options) {
|
|
|
344
509
|
apiKey: String(task.context.apiKey || ""),
|
|
345
510
|
systemPrompt: String(task.context.systemPrompt || ""),
|
|
346
511
|
reasoningEffort: String(task.context.reasoningEffort || ""),
|
|
512
|
+
ossPresignUrl: task.context.ossPresignUrl,
|
|
347
513
|
};
|
|
348
514
|
let params = isRecord(task.context.params) ? { ...task.context.params, taskAction: "create" } : { taskAction: "create" };
|
|
349
515
|
let result = await runScript(task.script, { ...common, params }, request, scriptFetch, options.signal, timeoutMs);
|
|
@@ -0,0 +1,102 @@
|
|
|
1
|
+
import { createRequire } from "node:module";
|
|
2
|
+
|
|
3
|
+
const require = createRequire(import.meta.url);
|
|
4
|
+
|
|
5
|
+
/**
|
|
6
|
+
* 尽量拿到一个可用的 SQLite 实现,按优先级:
|
|
7
|
+
* 1. better-sqlite3 —— Node 18/20/22 有预编译产物,装上即用
|
|
8
|
+
* 2. node:sqlite —— Node 22.5+ 内置,不需要任何依赖
|
|
9
|
+
* 都不可用时返回 null,由调用方降级为内存存储,保证代理仍能启动。
|
|
10
|
+
*
|
|
11
|
+
* 返回的对象只暴露 task-store 用到的最小接口:
|
|
12
|
+
* pragma(sql) / exec(sql) / prepare(sql).run|get|all / transaction(fn) / close()
|
|
13
|
+
*/
|
|
14
|
+
|
|
15
|
+
export function openSqlite(file) {
|
|
16
|
+
const better = loadBetterSqlite3();
|
|
17
|
+
if (better) {
|
|
18
|
+
try {
|
|
19
|
+
return { db: new better(file), backend: "better-sqlite3" };
|
|
20
|
+
} catch {
|
|
21
|
+
// 二进制与当前 Node ABI 不匹配等情况,继续尝试内置实现
|
|
22
|
+
}
|
|
23
|
+
}
|
|
24
|
+
|
|
25
|
+
const nodeSqlite = loadNodeSqlite();
|
|
26
|
+
if (nodeSqlite) {
|
|
27
|
+
try {
|
|
28
|
+
return { db: new NodeSqliteAdapter(file, nodeSqlite), backend: "node:sqlite" };
|
|
29
|
+
} catch {
|
|
30
|
+
// 内置实现不可用,走内存降级
|
|
31
|
+
}
|
|
32
|
+
}
|
|
33
|
+
|
|
34
|
+
return null;
|
|
35
|
+
}
|
|
36
|
+
|
|
37
|
+
function loadBetterSqlite3() {
|
|
38
|
+
try {
|
|
39
|
+
const mod = require("better-sqlite3");
|
|
40
|
+
return typeof mod === "function" ? mod : mod.default;
|
|
41
|
+
} catch {
|
|
42
|
+
return null;
|
|
43
|
+
}
|
|
44
|
+
}
|
|
45
|
+
|
|
46
|
+
function loadNodeSqlite() {
|
|
47
|
+
try {
|
|
48
|
+
const mod = require("node:sqlite");
|
|
49
|
+
return mod && mod.DatabaseSync ? mod : null;
|
|
50
|
+
} catch {
|
|
51
|
+
return null;
|
|
52
|
+
}
|
|
53
|
+
}
|
|
54
|
+
|
|
55
|
+
/** 把 node:sqlite 的 DatabaseSync 适配成 better-sqlite3 的调用习惯 */
|
|
56
|
+
class NodeSqliteAdapter {
|
|
57
|
+
#db;
|
|
58
|
+
|
|
59
|
+
constructor(file, { DatabaseSync }) {
|
|
60
|
+
this.#db = new DatabaseSync(file);
|
|
61
|
+
}
|
|
62
|
+
|
|
63
|
+
pragma(statement) {
|
|
64
|
+
this.#db.exec(`PRAGMA ${statement}`);
|
|
65
|
+
}
|
|
66
|
+
|
|
67
|
+
exec(sql) {
|
|
68
|
+
this.#db.exec(sql);
|
|
69
|
+
}
|
|
70
|
+
|
|
71
|
+
prepare(sql) {
|
|
72
|
+
const statement = this.#db.prepare(sql);
|
|
73
|
+
return {
|
|
74
|
+
run: (...args) => statement.run(...args),
|
|
75
|
+
get: (...args) => statement.get(...args),
|
|
76
|
+
all: (...args) => statement.all(...args),
|
|
77
|
+
};
|
|
78
|
+
}
|
|
79
|
+
|
|
80
|
+
transaction(fn) {
|
|
81
|
+
const db = this.#db;
|
|
82
|
+
return (...args) => {
|
|
83
|
+
db.exec("BEGIN");
|
|
84
|
+
try {
|
|
85
|
+
const result = fn(...args);
|
|
86
|
+
db.exec("COMMIT");
|
|
87
|
+
return result;
|
|
88
|
+
} catch (error) {
|
|
89
|
+
try {
|
|
90
|
+
db.exec("ROLLBACK");
|
|
91
|
+
} catch {
|
|
92
|
+
// 事务可能已经失效,保留原始错误
|
|
93
|
+
}
|
|
94
|
+
throw error;
|
|
95
|
+
}
|
|
96
|
+
};
|
|
97
|
+
}
|
|
98
|
+
|
|
99
|
+
close() {
|
|
100
|
+
this.#db.close();
|
|
101
|
+
}
|
|
102
|
+
}
|
package/task-store.js
CHANGED
|
@@ -1,15 +1,17 @@
|
|
|
1
1
|
import { mkdirSync } from "node:fs";
|
|
2
|
-
import { createRequire } from "node:module";
|
|
3
2
|
import path from "node:path";
|
|
4
|
-
|
|
5
|
-
|
|
6
|
-
const Database = require("better-sqlite3");
|
|
3
|
+
import { openSqlite } from "./sqlite-backend.js";
|
|
4
|
+
import { createMemoryTaskStore } from "./memory-task-store.js";
|
|
7
5
|
|
|
8
6
|
export function createTaskStore(dataDir) {
|
|
9
7
|
mkdirSync(dataDir, { recursive: true, mode: 0o700 });
|
|
10
8
|
const resultDir = path.join(dataDir, "results");
|
|
11
9
|
mkdirSync(resultDir, { recursive: true, mode: 0o700 });
|
|
12
|
-
|
|
10
|
+
|
|
11
|
+
const opened = openSqlite(path.join(dataDir, "tasks.sqlite"));
|
|
12
|
+
if (!opened) return createMemoryTaskStore(resultDir);
|
|
13
|
+
const { db, backend } = opened;
|
|
14
|
+
|
|
13
15
|
db.pragma("journal_mode = WAL");
|
|
14
16
|
db.pragma("synchronous = FULL");
|
|
15
17
|
db.exec(`
|
|
@@ -100,6 +102,8 @@ export function createTaskStore(dataDir) {
|
|
|
100
102
|
|
|
101
103
|
return {
|
|
102
104
|
resultDir,
|
|
105
|
+
backend,
|
|
106
|
+
persistent: true,
|
|
103
107
|
create(id, now, expiresAt) {
|
|
104
108
|
insert.run({ id, now, expiresAt, clientRequestId: null, kind: "request", source: null, metadataJson: null });
|
|
105
109
|
},
|