oasis_test_v2 2.2.118 → 2.2.119
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/index.js +689 -344
- package/package.json +2 -2
package/dist/index.js
CHANGED
|
@@ -756,7 +756,7 @@ function isDispatchFenced(error3) {
|
|
|
756
756
|
function isCompanyMembersLockLost(error3) {
|
|
757
757
|
return typeof error3 === "object" && error3 !== null && error3.companyMembersLockLost === true;
|
|
758
758
|
}
|
|
759
|
-
var SeqConflictError, BlobNotFoundError, BlobRangeNotSatisfiableError, BlobBackendError, BlobTooLargeError, BlobIntegrityError, AssistantBindConflictError, TraceSeqConflictError, DispatchUndecidableError, DispatchFencedError, CompanyMembersLockLostError;
|
|
759
|
+
var SeqConflictError, BlobNotFoundError, BlobRangeNotSatisfiableError, BlobBackendError, BlobTooLargeError, BlobSweepAbortedError, BlobIntegrityError, AssistantBindConflictError, TraceSeqConflictError, DispatchUndecidableError, DispatchFencedError, CompanyMembersLockLostError;
|
|
760
760
|
var init_errors = __esm({
|
|
761
761
|
"../contract/src/errors.ts"() {
|
|
762
762
|
"use strict";
|
|
@@ -802,6 +802,15 @@ var init_errors = __esm({
|
|
|
802
802
|
}
|
|
803
803
|
name = "BlobTooLargeError";
|
|
804
804
|
};
|
|
805
|
+
BlobSweepAbortedError = class extends Error {
|
|
806
|
+
constructor(candidateRows, totalRows, maxDeleteRatio) {
|
|
807
|
+
super(`blob sweep aborted: ${candidateRows}/${totalRows} rows would be deleted, above maxDeleteRatio ${maxDeleteRatio}; nothing deleted`);
|
|
808
|
+
this.candidateRows = candidateRows;
|
|
809
|
+
this.totalRows = totalRows;
|
|
810
|
+
this.maxDeleteRatio = maxDeleteRatio;
|
|
811
|
+
}
|
|
812
|
+
name = "BlobSweepAbortedError";
|
|
813
|
+
};
|
|
805
814
|
BlobIntegrityError = class extends Error {
|
|
806
815
|
constructor(expectedHash, actualHash) {
|
|
807
816
|
super(`blob hash mismatch: expected ${expectedHash}, got ${actualHash}`);
|
|
@@ -869,6 +878,24 @@ var init_errors = __esm({
|
|
|
869
878
|
});
|
|
870
879
|
|
|
871
880
|
// ../contract/src/storage.ts
|
|
881
|
+
function planBlobSweep(entries, reachable, opts = {}, nowMs = Date.now()) {
|
|
882
|
+
const cutoff = nowMs - Math.max(0, opts.graceMs ?? 0);
|
|
883
|
+
const candidates = entries.filter((e5) => !reachable.has(e5.hash) && (opts.graceMs ? e5.putAtMs < cutoff : true));
|
|
884
|
+
const report = {
|
|
885
|
+
dryRun: opts.dryRun === true,
|
|
886
|
+
totalRows: entries.length,
|
|
887
|
+
candidateRows: candidates.length,
|
|
888
|
+
candidateBytes: candidates.reduce((n3, e5) => n3 + e5.size, 0),
|
|
889
|
+
deletedRows: 0,
|
|
890
|
+
deletedBytes: 0,
|
|
891
|
+
orphansFound: 0,
|
|
892
|
+
orphansDeleted: 0
|
|
893
|
+
};
|
|
894
|
+
if (opts.maxDeleteRatio !== void 0 && entries.length > 0 && candidates.length / entries.length > opts.maxDeleteRatio) {
|
|
895
|
+
throw new BlobSweepAbortedError(candidates.length, entries.length, opts.maxDeleteRatio);
|
|
896
|
+
}
|
|
897
|
+
return { report, toDelete: opts.dryRun ? [] : candidates };
|
|
898
|
+
}
|
|
872
899
|
async function putBlobStream(store, source, opts = {}) {
|
|
873
900
|
if (store.putStream) return store.putStream(source, opts);
|
|
874
901
|
const parts = [];
|
|
@@ -2819,6 +2846,52 @@ var init_file_limits2 = __esm({
|
|
|
2819
2846
|
}
|
|
2820
2847
|
});
|
|
2821
2848
|
|
|
2849
|
+
// ../server/src/raw-upload.ts
|
|
2850
|
+
function isRawUploadRequest(method, pathname) {
|
|
2851
|
+
return RAW_UPLOAD_ROUTES.some((r5) => r5.method === method && r5.path.test(pathname));
|
|
2852
|
+
}
|
|
2853
|
+
function rawUploadName(req) {
|
|
2854
|
+
const raw = req.headers["x-oasis-filename"];
|
|
2855
|
+
const value2 = Array.isArray(raw) ? raw[0] : raw;
|
|
2856
|
+
if (!value2) return void 0;
|
|
2857
|
+
let name;
|
|
2858
|
+
try {
|
|
2859
|
+
name = decodeURIComponent(value2);
|
|
2860
|
+
} catch {
|
|
2861
|
+
return void 0;
|
|
2862
|
+
}
|
|
2863
|
+
return name.trim() || void 0;
|
|
2864
|
+
}
|
|
2865
|
+
function rawUploadContentType(req) {
|
|
2866
|
+
const value2 = req.headers["content-type"]?.trim();
|
|
2867
|
+
return value2 && value2 !== "application/octet-stream" ? value2 : void 0;
|
|
2868
|
+
}
|
|
2869
|
+
async function putRawUpload(store, req, opts) {
|
|
2870
|
+
const code2 = opts.tooLargeCode ?? "FILE_TOO_LARGE";
|
|
2871
|
+
const tooLarge = () => new ApiError(413, code2, `\u6587\u4EF6\u8D85\u8FC7 ${opts.maxBytes / 1024 / 1024} MiB \u4E0A\u9650`);
|
|
2872
|
+
const declared = req.headers["content-length"];
|
|
2873
|
+
if (declared !== void 0 && Number(declared) > opts.maxBytes) throw tooLarge();
|
|
2874
|
+
try {
|
|
2875
|
+
return await putBlobStream(store, req.iterator({ destroyOnReturn: false }), { maxBytes: opts.maxBytes });
|
|
2876
|
+
} catch (err) {
|
|
2877
|
+
if (err instanceof BlobTooLargeError) throw tooLarge();
|
|
2878
|
+
throw err;
|
|
2879
|
+
}
|
|
2880
|
+
}
|
|
2881
|
+
var RAW_UPLOAD_ROUTES;
|
|
2882
|
+
var init_raw_upload = __esm({
|
|
2883
|
+
"../server/src/raw-upload.ts"() {
|
|
2884
|
+
"use strict";
|
|
2885
|
+
init_src();
|
|
2886
|
+
init_router();
|
|
2887
|
+
RAW_UPLOAD_ROUTES = [
|
|
2888
|
+
{ method: "PUT", path: /^\/api\/chat\/attachments\/raw$/ },
|
|
2889
|
+
{ method: "POST", path: /^\/api\/workorders\/[^/]+\/files\/raw$/ },
|
|
2890
|
+
{ method: "POST", path: /^\/api\/projects\/[^/]+\/files\/raw$/ }
|
|
2891
|
+
];
|
|
2892
|
+
}
|
|
2893
|
+
});
|
|
2894
|
+
|
|
2822
2895
|
// ../server/src/chat-system-message.ts
|
|
2823
2896
|
async function appendChatMessageOnce(store, m4) {
|
|
2824
2897
|
const opts = { source: "system" };
|
|
@@ -153854,15 +153927,34 @@ async function reachableBlobs(refs, blobs) {
|
|
|
153854
153927
|
}
|
|
153855
153928
|
return reachable;
|
|
153856
153929
|
}
|
|
153857
|
-
|
|
153930
|
+
function projectFileRefs(projects) {
|
|
153931
|
+
return async () => {
|
|
153932
|
+
const out = /* @__PURE__ */ new Set();
|
|
153933
|
+
for (const p4 of await projects.listProjects()) {
|
|
153934
|
+
for (const f6 of await projects.listProjectFiles(p4.id)) if (f6.blobId) out.add(f6.blobId);
|
|
153935
|
+
for (const loc of await projects.listContentLocations(p4.id)) if (loc.blob?.blobId) out.add(loc.blob.blobId);
|
|
153936
|
+
}
|
|
153937
|
+
return out;
|
|
153938
|
+
};
|
|
153939
|
+
}
|
|
153940
|
+
async function gc(refs, blobs, opts = {}) {
|
|
153858
153941
|
const reachable = await reachableBlobs(refs, blobs);
|
|
153859
|
-
await
|
|
153860
|
-
|
|
153942
|
+
for (const extra of opts.extraRefs ?? []) for (const h5 of await extra()) reachable.add(h5);
|
|
153943
|
+
const report = await blobs.sweep(reachable, {
|
|
153944
|
+
dryRun: opts.dryRun === true,
|
|
153945
|
+
graceMs: opts.graceMs ?? GC_DEFAULT_GRACE_MS,
|
|
153946
|
+
maxDeleteRatio: opts.maxDeleteRatio ?? GC_DEFAULT_MAX_DELETE_RATIO,
|
|
153947
|
+
...opts.orphans ? { orphans: true } : {}
|
|
153948
|
+
});
|
|
153949
|
+
return { reachableCount: reachable.size, report: report ?? null };
|
|
153861
153950
|
}
|
|
153951
|
+
var GC_DEFAULT_GRACE_MS, GC_DEFAULT_MAX_DELETE_RATIO;
|
|
153862
153952
|
var init_gc = __esm({
|
|
153863
153953
|
"../core/src/gc.ts"() {
|
|
153864
153954
|
"use strict";
|
|
153865
153955
|
init_manifest_store();
|
|
153956
|
+
GC_DEFAULT_GRACE_MS = 24 * 60 * 60 * 1e3;
|
|
153957
|
+
GC_DEFAULT_MAX_DELETE_RATIO = 0.2;
|
|
153866
153958
|
}
|
|
153867
153959
|
});
|
|
153868
153960
|
|
|
@@ -213973,11 +214065,30 @@ ${acceptanceCriteria.map((c5) => `- ${c5}`).join("\n")}`] : []
|
|
|
213973
214065
|
return { message: "tool_call \u5DF2\u7559\u75D5" };
|
|
213974
214066
|
}
|
|
213975
214067
|
case "gc": {
|
|
214068
|
+
const dryRun = args["dryRun"] === true;
|
|
214069
|
+
const orphans = args["orphans"] === true;
|
|
214070
|
+
const ratio = (key2, lo, hi) => {
|
|
214071
|
+
const v2 = args[key2];
|
|
214072
|
+
if (v2 === void 0 || v2 === null) return void 0;
|
|
214073
|
+
if (typeof v2 !== "number" || !Number.isFinite(v2) || v2 < lo || v2 > hi) throw new Error(`\u53C2\u6570 ${key2} \u5FC5\u987B\u662F ${lo}\u2013${hi} \u4E4B\u95F4\u7684\u6570`);
|
|
214074
|
+
return v2;
|
|
214075
|
+
};
|
|
214076
|
+
const graceHours = ratio("graceHours", 0, 24 * 365);
|
|
214077
|
+
const maxDeleteRatio = ratio("maxDeleteRatio", 0, 1);
|
|
214078
|
+
if (!ctx.projectState && !dryRun) throw new Error("gc\uFF1A\u62FF\u4E0D\u5230\u672C\u516C\u53F8\u9879\u76EE\u5E93\uFF0C\u9879\u76EE\u6587\u4EF6\u65E0\u6CD5\u8FDB\u53EF\u8FBE\u96C6\uFF1B\u53EA\u5141\u8BB8 --dry-run");
|
|
213976
214079
|
const result = await gc(async () => {
|
|
213977
214080
|
if (!engineStore) return [];
|
|
213978
214081
|
return engineStore.transaction((tx) => tx.listAllArtifactRefs());
|
|
213979
|
-
}, blobs
|
|
213980
|
-
|
|
214082
|
+
}, blobs, {
|
|
214083
|
+
...ctx.projectState ? { extraRefs: [projectFileRefs(ctx.projectState)] } : {},
|
|
214084
|
+
dryRun,
|
|
214085
|
+
orphans,
|
|
214086
|
+
...graceHours !== void 0 ? { graceMs: graceHours * 36e5 } : {},
|
|
214087
|
+
...maxDeleteRatio !== void 0 ? { maxDeleteRatio } : {}
|
|
214088
|
+
});
|
|
214089
|
+
const r5 = result.report;
|
|
214090
|
+
const summary = !r5 ? "\uFF08\u5B58\u50A8\u672A\u56DE\u62A5\u6E05\u626B\u6570\uFF09" : dryRun ? `dry-run\uFF1A\u5171 ${r5.totalRows} \u884C\uFF0C\u4F1A\u5220 ${r5.candidateRows} \u884C / ${r5.candidateBytes} \u5B57\u8282${orphans ? `\uFF0C\u5B64\u513F\u5BF9\u8C61 ${r5.orphansFound} \u4E2A` : ""}\uFF1B\u672A\u5220\u9664\u4EFB\u4F55\u5185\u5BB9` : `\u5171 ${r5.totalRows} \u884C\uFF0C\u5220\u9664 ${r5.deletedRows} \u884C / ${r5.deletedBytes} \u5B57\u8282${orphans ? `\uFF0C\u5B64\u513F\u5BF9\u8C61 ${r5.orphansDeleted} \u4E2A` : ""}`;
|
|
214091
|
+
return { message: `GC \u5B8C\u6210\uFF1A\u4E8B\u5B9E\u67DC\u53EF\u8FBE blob ${result.reachableCount} \u4E2A\uFF1B${summary}`, data: result };
|
|
213981
214092
|
}
|
|
213982
214093
|
default:
|
|
213983
214094
|
throw new Error(`\u672A\u77E5\u547D\u4EE4: ${command6}`);
|
|
@@ -215099,13 +215210,15 @@ async function startOasisServer(opts) {
|
|
|
215099
215210
|
void (async () => {
|
|
215100
215211
|
try {
|
|
215101
215212
|
res.setHeader("access-control-allow-origin", "*");
|
|
215102
|
-
res.setHeader("access-control-allow-methods", "GET,POST,PATCH,DELETE,OPTIONS");
|
|
215103
|
-
res.setHeader("access-control-allow-headers", "authorization,content-type");
|
|
215213
|
+
res.setHeader("access-control-allow-methods", "GET,POST,PUT,PATCH,DELETE,OPTIONS");
|
|
215214
|
+
res.setHeader("access-control-allow-headers", "authorization,content-type,x-oasis-company,x-oasis-filename");
|
|
215104
215215
|
if (req.method === "OPTIONS") {
|
|
215105
215216
|
res.writeHead(204).end();
|
|
215106
215217
|
return;
|
|
215107
215218
|
}
|
|
215108
215219
|
const url = new URL(req.url ?? "/", "http://localhost");
|
|
215220
|
+
const rawUpload = isRawUploadRequest(req.method, url.pathname);
|
|
215221
|
+
if (rawUpload) res.setHeader("connection", "close");
|
|
215109
215222
|
if (url.pathname === "/healthz" && url.searchParams.get("deep") === "1") {
|
|
215110
215223
|
const db = opts.deepHealth ? await opts.deepHealth() : { ok: true, db: "none" };
|
|
215111
215224
|
res.writeHead(db.ok ? 200 : 503, { "content-type": "application/json" }).end(JSON.stringify({ ok: db.ok, ...getBuildInfo(), db }));
|
|
@@ -215283,13 +215396,13 @@ async function startOasisServer(opts) {
|
|
|
215283
215396
|
const requestMaxBytes = fileUploadRequestMaxBytes(fileLimits);
|
|
215284
215397
|
const chunks = [];
|
|
215285
215398
|
let requestBytes = 0;
|
|
215286
|
-
const declared = Number(req.headers["content-length"] ?? 0);
|
|
215399
|
+
const declared = rawUpload ? 0 : Number(req.headers["content-length"] ?? 0);
|
|
215287
215400
|
if (declared > requestMaxBytes) {
|
|
215288
215401
|
req.resume();
|
|
215289
215402
|
res.writeHead(413, { "content-type": "application/json" }).end(JSON.stringify({ error: { code: "UPLOAD_TOO_LARGE", message: "\u4E0A\u4F20\u8BF7\u6C42\u8D85\u8FC7\u901A\u9053\u5BB9\u91CF" } }));
|
|
215290
215403
|
return;
|
|
215291
215404
|
}
|
|
215292
|
-
for await (const chunk of req) {
|
|
215405
|
+
if (!rawUpload) for await (const chunk of req) {
|
|
215293
215406
|
requestBytes += chunk.length;
|
|
215294
215407
|
if (requestBytes > requestMaxBytes) {
|
|
215295
215408
|
res.writeHead(413, { "content-type": "application/json" }).end(JSON.stringify({ error: { code: "UPLOAD_TOO_LARGE", message: "\u4E0A\u4F20\u8BF7\u6C42\u8D85\u8FC7\u901A\u9053\u5BB9\u91CF" } }));
|
|
@@ -215536,7 +215649,7 @@ async function startOasisServer(opts) {
|
|
|
215536
215649
|
}
|
|
215537
215650
|
const chatAnchorMessageId = await anchorMessageIdFor(chatOrigin, chatStoreNowOrNull);
|
|
215538
215651
|
const spawnArgs = body2.args;
|
|
215539
|
-
const needsProject = body2.command === "spawn" && (spawnArgs?.["createdVia"] === "seed" || spawnArgs?.["project_id"] !== void 0 || spawnArgs?.["projectId"] !== void 0);
|
|
215652
|
+
const needsProject = body2.command === "gc" || body2.command === "spawn" && (spawnArgs?.["createdVia"] === "seed" || spawnArgs?.["project_id"] !== void 0 || spawnArgs?.["projectId"] !== void 0);
|
|
215540
215653
|
const assertCmdPlanMembers = (plan) => assertPlanMembers(plan).catch((err) => {
|
|
215541
215654
|
if (err instanceof NotCompanyMemberError) throw new ApiError(err.status, err.code, err.message);
|
|
215542
215655
|
throw err;
|
|
@@ -216784,6 +216897,21 @@ async function startOasisServer(opts) {
|
|
|
216784
216897
|
}
|
|
216785
216898
|
return;
|
|
216786
216899
|
}
|
|
216900
|
+
if (url.pathname === "/api/chat/attachments/raw" && req.method === "PUT" && opts.assets) {
|
|
216901
|
+
const uploadKind = url.searchParams.get("purpose");
|
|
216902
|
+
const maxBytes = uploadKind === "knowledge" ? fileLimits.knowledgeBytes : uploadKind === "cli" ? fileLimits.cliBytes : fileLimits.attachmentBytes;
|
|
216903
|
+
try {
|
|
216904
|
+
const out = await putRawUpload(opts.assets, req, { maxBytes, tooLargeCode: "ATTACHMENT_TOO_LARGE" });
|
|
216905
|
+
res.writeHead(201, { "content-type": "application/json" }).end(JSON.stringify({ blobRef: out.hash }));
|
|
216906
|
+
} catch (err) {
|
|
216907
|
+
const e5 = err instanceof ApiError ? err : null;
|
|
216908
|
+
if (!e5) console.error(`[chat/attachments/raw] ${err instanceof Error ? err.message : String(err)}`);
|
|
216909
|
+
res.writeHead(e5?.status ?? 500, { "content-type": "application/json" }).end(JSON.stringify({
|
|
216910
|
+
error: e5 ? { code: e5.code, message: e5.status === 413 ? `\u9644\u4EF6\u8D85\u8FC7 ${maxBytes / 1024 / 1024}MB \u4E0A\u9650` : e5.message } : { code: "ATTACHMENT_UPLOAD_FAILED", message: "\u9644\u4EF6\u4FDD\u5B58\u5931\u8D25\uFF0C\u8BF7\u91CD\u8BD5" }
|
|
216911
|
+
}));
|
|
216912
|
+
}
|
|
216913
|
+
return;
|
|
216914
|
+
}
|
|
216787
216915
|
if (url.pathname === "/api/chat/attachments" && req.method === "POST" && opts.uploadChatAttachment) {
|
|
216788
216916
|
try {
|
|
216789
216917
|
const body2 = JSON.parse(rawBody);
|
|
@@ -219402,6 +219530,7 @@ var init_server3 = __esm({
|
|
|
219402
219530
|
"use strict";
|
|
219403
219531
|
init_src();
|
|
219404
219532
|
init_file_limits2();
|
|
219533
|
+
init_raw_upload();
|
|
219405
219534
|
http = __toESM(require("node:http"), 1);
|
|
219406
219535
|
import_node_crypto44 = require("node:crypto");
|
|
219407
219536
|
import_node_stream3 = require("node:stream");
|
|
@@ -223172,7 +223301,7 @@ var init_blob_object_backend = __esm({
|
|
|
223172
223301
|
for await (const file of walk(base)) {
|
|
223173
223302
|
const st = await fs25.promises.stat(file).catch(() => null);
|
|
223174
223303
|
if (!st) continue;
|
|
223175
|
-
yield { key: path24.relative(this.root, file).split(path24.sep).join("/"), size: st.size, mtimeMs: st.mtimeMs };
|
|
223304
|
+
yield { key: path24.relative(this.root, file).split(path24.sep).join("/"), size: st.size, mtimeMs: Math.max(st.mtimeMs, st.ctimeMs) };
|
|
223176
223305
|
}
|
|
223177
223306
|
}
|
|
223178
223307
|
};
|
|
@@ -225338,15 +225467,29 @@ var init_service4 = __esm({
|
|
|
225338
225467
|
}
|
|
225339
225468
|
/** 上传一个项目普通文件:内容入 blob 库,记一条元信息旁存(非 oplog 事实)。 */
|
|
225340
225469
|
async addProjectFile(input) {
|
|
225341
|
-
|
|
225470
|
+
const blobs = this.projectFileBlobs();
|
|
225342
225471
|
assertFileBytes(input.bytes);
|
|
225343
|
-
const blobId = await
|
|
225472
|
+
const blobId = await blobs.put(input.bytes);
|
|
225473
|
+
const { bytes: bytes2, ...meta } = input;
|
|
225474
|
+
return this.recordProjectFile({ ...meta, blobId, size: bytes2.byteLength });
|
|
225475
|
+
}
|
|
225476
|
+
/**
|
|
225477
|
+
* 项目文件的字节柜。原始字节上传(`POST /api/projects/:id/files/raw`,ADR 0638 S4)由路由把请求体
|
|
225478
|
+
* 流式写进这里,再调 {@link recordProjectFile} 记行。
|
|
225479
|
+
*/
|
|
225480
|
+
projectFileBlobs() {
|
|
225481
|
+
if (!this.blobs) throw new Error("\u9879\u76EE\u6587\u4EF6\u4E0A\u4F20\u9700\u8981 blob \u5E93\uFF08domain \u672A\u6CE8\u5165 blobs\uFF09");
|
|
225482
|
+
return this.blobs;
|
|
225483
|
+
}
|
|
225484
|
+
/** 字节已进柜之后的那一半:记一条项目文件元信息。 */
|
|
225485
|
+
async recordProjectFile(input) {
|
|
225486
|
+
const { blobId, size } = input;
|
|
225344
225487
|
const file = {
|
|
225345
225488
|
projectId: input.projectId,
|
|
225346
225489
|
fileId: `pf_${Date.now().toString(36)}${Math.random().toString(36).slice(2, 8)}`,
|
|
225347
225490
|
name: input.name,
|
|
225348
225491
|
...input.path !== void 0 ? { path: input.path } : {},
|
|
225349
|
-
size
|
|
225492
|
+
size,
|
|
225350
225493
|
blobId,
|
|
225351
225494
|
...input.contentType !== void 0 ? { contentType: input.contentType } : {},
|
|
225352
225495
|
...input.uploadedBy !== void 0 ? { uploadedBy: input.uploadedBy } : {},
|
|
@@ -226919,13 +227062,25 @@ var init_dev_store = __esm({
|
|
|
226919
227062
|
async put(bytes2) {
|
|
226920
227063
|
const hash3 = (0, import_node_crypto51.createHash)("sha256").update(bytes2).digest("hex");
|
|
226921
227064
|
const existing = fs26.statSync(this.fileOf(hash3), { throwIfNoEntry: false });
|
|
226922
|
-
if (existing?.isFile() && existing.size === bytes2.byteLength)
|
|
227065
|
+
if (existing?.isFile() && existing.size === bytes2.byteLength) {
|
|
227066
|
+
this.touch(hash3);
|
|
227067
|
+
return hash3;
|
|
227068
|
+
}
|
|
226923
227069
|
return (await this.putStream(singleChunk(bytes2))).hash;
|
|
226924
227070
|
}
|
|
227071
|
+
/** 重复 put 也刷新 mtime:GC 宽限期按它算(同 PG 的 `last_put_at`)。 */
|
|
227072
|
+
touch(hash3) {
|
|
227073
|
+
const now = /* @__PURE__ */ new Date();
|
|
227074
|
+
try {
|
|
227075
|
+
fs26.utimesSync(this.fileOf(hash3), now, now);
|
|
227076
|
+
} catch {
|
|
227077
|
+
}
|
|
227078
|
+
}
|
|
226925
227079
|
/** 流式写(ADR 0638 §3.2):边收边算 hash,超 `maxBytes` / hash 不符删残件抛错。 */
|
|
226926
227080
|
async putStream(source, opts) {
|
|
226927
227081
|
const staged = await stageBlob(this.tmpDir, source, opts);
|
|
226928
227082
|
await commitStagedFile(staged, this.fileOf(staged.hash), 420);
|
|
227083
|
+
this.touch(staged.hash);
|
|
226929
227084
|
const contentType = sniffContentType(staged.head);
|
|
226930
227085
|
return { hash: staged.hash, size: staged.size, ...contentType ? { contentType } : {} };
|
|
226931
227086
|
}
|
|
@@ -226981,10 +227136,20 @@ var init_dev_store = __esm({
|
|
|
226981
227136
|
async has(hash3) {
|
|
226982
227137
|
return BLOB_NAME.test(hash3) && fs26.statSync(this.fileOf(hash3), { throwIfNoEntry: false })?.isFile() === true;
|
|
226983
227138
|
}
|
|
226984
|
-
async sweep(reachable) {
|
|
227139
|
+
async sweep(reachable, opts = {}) {
|
|
227140
|
+
const entries = [];
|
|
226985
227141
|
for (const name of fs26.readdirSync(this.dir)) {
|
|
226986
|
-
if (BLOB_NAME.test(name)
|
|
227142
|
+
if (!BLOB_NAME.test(name)) continue;
|
|
227143
|
+
const st = fs26.statSync(this.fileOf(name), { throwIfNoEntry: false });
|
|
227144
|
+
if (st?.isFile()) entries.push({ hash: name, size: st.size, putAtMs: st.mtimeMs });
|
|
227145
|
+
}
|
|
227146
|
+
const { report, toDelete } = planBlobSweep(entries, reachable, opts);
|
|
227147
|
+
for (const e5 of toDelete) {
|
|
227148
|
+
fs26.rmSync(this.fileOf(e5.hash), { force: true });
|
|
227149
|
+
report.deletedRows++;
|
|
227150
|
+
report.deletedBytes += e5.size;
|
|
226987
227151
|
}
|
|
227152
|
+
return report;
|
|
226988
227153
|
}
|
|
226989
227154
|
};
|
|
226990
227155
|
MUTATORS = [
|
|
@@ -229104,6 +229269,18 @@ async function copySessionOutputs(deps) {
|
|
|
229104
229269
|
result.skipped.push({ path: file.path, code: "BUDGET" });
|
|
229105
229270
|
continue;
|
|
229106
229271
|
}
|
|
229272
|
+
const release2 = file.size > OUTPUT_COPY_SERIAL_BYTES && deps.acquireLargeFileSlot ? await deps.acquireLargeFileSlot() : void 0;
|
|
229273
|
+
try {
|
|
229274
|
+
await copyOne(file);
|
|
229275
|
+
} finally {
|
|
229276
|
+
release2?.();
|
|
229277
|
+
}
|
|
229278
|
+
}
|
|
229279
|
+
async function copyOne(file) {
|
|
229280
|
+
if (Date.now() >= deadline) {
|
|
229281
|
+
result.skipped.push({ path: file.path, code: "BUDGET" });
|
|
229282
|
+
return;
|
|
229283
|
+
}
|
|
229107
229284
|
let read3;
|
|
229108
229285
|
try {
|
|
229109
229286
|
read3 = await deps.bridge.read(
|
|
@@ -229113,20 +229290,20 @@ async function copySessionOutputs(deps) {
|
|
|
229113
229290
|
);
|
|
229114
229291
|
} catch {
|
|
229115
229292
|
result.skipped.push({ path: file.path, code: "READ_FAILED" });
|
|
229116
|
-
|
|
229293
|
+
return;
|
|
229117
229294
|
}
|
|
229118
229295
|
if (!read3.ok) {
|
|
229119
229296
|
result.skipped.push({ path: file.path, code: read3.code === "TOO_LARGE" ? "TOO_LARGE" : "READ_FAILED" });
|
|
229120
|
-
|
|
229297
|
+
return;
|
|
229121
229298
|
}
|
|
229122
229299
|
const bytes2 = Buffer.from(read3.contentBase64, "base64");
|
|
229123
229300
|
if (bytes2.length > limits.maxFileBytes) {
|
|
229124
229301
|
result.skipped.push({ path: file.path, code: "TOO_LARGE" });
|
|
229125
|
-
|
|
229302
|
+
return;
|
|
229126
229303
|
}
|
|
229127
229304
|
if (bytes2.length !== file.size || read3.mtime && read3.mtime !== file.mtime) {
|
|
229128
229305
|
result.skipped.push({ path: file.path, code: "FILE_CHANGED" });
|
|
229129
|
-
|
|
229306
|
+
return;
|
|
229130
229307
|
}
|
|
229131
229308
|
const saved = existing?.[file.path];
|
|
229132
229309
|
if (saved && savedContentHash(saved) === (0, import_node_crypto53.createHash)("sha256").update(bytes2).digest("hex")) {
|
|
@@ -229134,14 +229311,14 @@ async function copySessionOutputs(deps) {
|
|
|
229134
229311
|
delete aligned.editedAt;
|
|
229135
229312
|
realigned.push(aligned);
|
|
229136
229313
|
result.unchanged.push(file.path);
|
|
229137
|
-
|
|
229314
|
+
return;
|
|
229138
229315
|
}
|
|
229139
229316
|
let blobRef;
|
|
229140
229317
|
try {
|
|
229141
|
-
blobRef = await deps.putBlob(
|
|
229318
|
+
blobRef = await deps.putBlob(bytes2);
|
|
229142
229319
|
} catch {
|
|
229143
229320
|
result.skipped.push({ path: file.path, code: "STORE_FAILED" });
|
|
229144
|
-
|
|
229321
|
+
return;
|
|
229145
229322
|
}
|
|
229146
229323
|
result.copied.push({
|
|
229147
229324
|
path: file.path,
|
|
@@ -229167,6 +229344,8 @@ async function copySessionOutputs(deps) {
|
|
|
229167
229344
|
}
|
|
229168
229345
|
function createOutputCopyScheduler(deps) {
|
|
229169
229346
|
const inFlight = /* @__PURE__ */ new Map();
|
|
229347
|
+
const largeFiles = createKeyedMutex();
|
|
229348
|
+
const acquireLargeFileSlot = () => largeFiles.acquire("large-output-copy");
|
|
229170
229349
|
return {
|
|
229171
229350
|
schedule(args) {
|
|
229172
229351
|
const { store, chatSessionId, companyId } = args;
|
|
@@ -229190,6 +229369,7 @@ function createOutputCopyScheduler(deps) {
|
|
|
229190
229369
|
readCopies,
|
|
229191
229370
|
mergeCopies,
|
|
229192
229371
|
putBlob: deps.putBlob,
|
|
229372
|
+
acquireLargeFileSlot,
|
|
229193
229373
|
...deps.limits ? { limits: deps.limits } : {}
|
|
229194
229374
|
});
|
|
229195
229375
|
const maxFileMiB = Math.round((deps.limits ?? OUTPUT_COPY_LIMITS).maxFileBytes / 1024 / 1024);
|
|
@@ -229226,17 +229406,19 @@ function createOutputCopyScheduler(deps) {
|
|
|
229226
229406
|
}
|
|
229227
229407
|
};
|
|
229228
229408
|
}
|
|
229229
|
-
var import_node_crypto53, OUTPUT_COPY_LIMITS;
|
|
229409
|
+
var import_node_crypto53, OUTPUT_COPY_LIMITS, OUTPUT_COPY_SERIAL_BYTES;
|
|
229230
229410
|
var init_outputs_copy = __esm({
|
|
229231
229411
|
"../server/src/domains/chat-sessions/outputs-copy.ts"() {
|
|
229232
229412
|
"use strict";
|
|
229233
229413
|
import_node_crypto53 = require("node:crypto");
|
|
229234
229414
|
init_src();
|
|
229415
|
+
init_keyed_mutex();
|
|
229235
229416
|
OUTPUT_COPY_LIMITS = {
|
|
229236
229417
|
maxFileBytes: FILE_CONTENT_MAX_BYTES,
|
|
229237
229418
|
maxFiles: 50,
|
|
229238
229419
|
maxTotalBytes: 2 * FILE_CONTENT_MAX_BYTES
|
|
229239
229420
|
};
|
|
229421
|
+
OUTPUT_COPY_SERIAL_BYTES = 20 * 1024 * 1024;
|
|
229240
229422
|
}
|
|
229241
229423
|
});
|
|
229242
229424
|
|
|
@@ -239360,6 +239542,24 @@ function projectsDomain(opts) {
|
|
|
239360
239542
|
});
|
|
239361
239543
|
return { status: 201, body: file };
|
|
239362
239544
|
});
|
|
239545
|
+
router.post("/api/projects/:id/files/raw", async (req) => {
|
|
239546
|
+
const name = rawUploadName(req.raw);
|
|
239547
|
+
if (!name) throw new ApiError(400, "BAD_REQUEST", "\u7F3A\u5C11 X-Oasis-Filename");
|
|
239548
|
+
const path50 = req.query.get("path") ?? void 0;
|
|
239549
|
+
const contentType = rawUploadContentType(req.raw);
|
|
239550
|
+
const service = await svc(req);
|
|
239551
|
+
const put3 = await putRawUpload(service.projectFileBlobs(), req.raw, { maxBytes: currentFileLimits().attachmentBytes });
|
|
239552
|
+
const file = await service.recordProjectFile({
|
|
239553
|
+
projectId: req.params.id,
|
|
239554
|
+
name,
|
|
239555
|
+
...path50 !== void 0 ? { path: path50 } : {},
|
|
239556
|
+
blobId: put3.hash,
|
|
239557
|
+
size: put3.size,
|
|
239558
|
+
...contentType !== void 0 ? { contentType } : {},
|
|
239559
|
+
uploadedBy: req.auth.actor
|
|
239560
|
+
});
|
|
239561
|
+
return { status: 201, body: file };
|
|
239562
|
+
});
|
|
239363
239563
|
router.put("/api/projects/:id/files/:fileId", async (req) => {
|
|
239364
239564
|
const b6 = req.body ?? {};
|
|
239365
239565
|
const base642 = b6.content_base64 ?? b6.contentBase64;
|
|
@@ -240122,6 +240322,8 @@ var init_routes5 = __esm({
|
|
|
240122
240322
|
"../server/src/domains/projects/routes.ts"() {
|
|
240123
240323
|
"use strict";
|
|
240124
240324
|
init_router();
|
|
240325
|
+
init_file_limits2();
|
|
240326
|
+
init_raw_upload();
|
|
240125
240327
|
init_src();
|
|
240126
240328
|
init_chat_files2();
|
|
240127
240329
|
init_chat_sessions();
|
|
@@ -246047,33 +246249,26 @@ function collabDomain(opts) {
|
|
|
246047
246249
|
}));
|
|
246048
246250
|
return { status: 200, body: { total: items.length, items } };
|
|
246049
246251
|
});
|
|
246050
|
-
|
|
246252
|
+
const resolveUploadTarget = async (req) => {
|
|
246051
246253
|
const { kernel, registry: registry2, artifacts } = await resolveCtx(req.auth.companyId);
|
|
246052
246254
|
const ws = req.params.id;
|
|
246053
246255
|
const exists = [...kernel.model.artifacts.values()].some((a5) => a5.workspace === ws);
|
|
246054
246256
|
if (!exists) {
|
|
246055
|
-
return { status: 404, body: { error: { code: "not_found", message: `\u5DE5\u5355\u4E0D\u5B58\u5728: ${ws}` } } };
|
|
246257
|
+
return { denied: { status: 404, body: { error: { code: "not_found", message: `\u5DE5\u5355\u4E0D\u5B58\u5728: ${ws}` } } } };
|
|
246056
246258
|
}
|
|
246057
246259
|
if (!opts.assets || !artifacts?.addWorkOrderFile) {
|
|
246058
246260
|
return {
|
|
246059
|
-
|
|
246060
|
-
|
|
246261
|
+
denied: {
|
|
246262
|
+
status: 501,
|
|
246263
|
+
body: { error: { code: "unavailable", message: "\u672C\u90E8\u7F72\u672A\u63A5\u5165\u4EFB\u52A1\u6587\u4EF6\u4E0A\u4F20\uFF08\u7F3A\u8D44\u4EA7\u67DC\u6216 addWorkOrderFile\uFF09" } }
|
|
246264
|
+
}
|
|
246061
246265
|
};
|
|
246062
246266
|
}
|
|
246063
|
-
|
|
246064
|
-
|
|
246065
|
-
|
|
246066
|
-
}
|
|
246067
|
-
const
|
|
246068
|
-
const contentType = typeof body2.contentType === "string" && body2.contentType.trim() ? body2.contentType.trim() : void 0;
|
|
246069
|
-
let bytes2;
|
|
246070
|
-
try {
|
|
246071
|
-
bytes2 = new Uint8Array(Buffer.from(body2.contentBase64, "base64"));
|
|
246072
|
-
} catch (err) {
|
|
246073
|
-
return { status: 400, body: { error: { code: "bad_request", message: `contentBase64 \u89E3\u7801\u5931\u8D25\uFF1A${err instanceof Error ? err.message : String(err)}` } } };
|
|
246074
|
-
}
|
|
246075
|
-
assertFileBytes(bytes2);
|
|
246076
|
-
const blobId = await opts.assets.put(bytes2);
|
|
246267
|
+
return { target: { ws, registry: registry2, assets: opts.assets, addWorkOrderFile: artifacts.addWorkOrderFile.bind(artifacts) } };
|
|
246268
|
+
};
|
|
246269
|
+
const linkUploadedFile = async (req, target, input) => {
|
|
246270
|
+
const { ws, registry: registry2, addWorkOrderFile } = target;
|
|
246271
|
+
const { blobId, name, size, contentType } = input;
|
|
246077
246272
|
const linkedAt = (/* @__PURE__ */ new Date()).toISOString();
|
|
246078
246273
|
const fileId = workOrderFileId(ws, blobId, name);
|
|
246079
246274
|
const file = {
|
|
@@ -246081,14 +246276,14 @@ function collabDomain(opts) {
|
|
|
246081
246276
|
fileId,
|
|
246082
246277
|
name,
|
|
246083
246278
|
blobId,
|
|
246084
|
-
size
|
|
246279
|
+
size,
|
|
246085
246280
|
...contentType ? { contentType } : {},
|
|
246086
246281
|
linkedAt,
|
|
246087
246282
|
// 上传者:项目页「创建人」列与预览副标据此署名(建单带入的那条路记的是建单发起人)。
|
|
246088
246283
|
uploadedBy: req.auth.actor
|
|
246089
246284
|
};
|
|
246090
246285
|
try {
|
|
246091
|
-
await
|
|
246286
|
+
await addWorkOrderFile(file);
|
|
246092
246287
|
} catch (err) {
|
|
246093
246288
|
return {
|
|
246094
246289
|
status: 500,
|
|
@@ -246101,7 +246296,7 @@ function collabDomain(opts) {
|
|
|
246101
246296
|
name,
|
|
246102
246297
|
kind: "file",
|
|
246103
246298
|
content_type: contentType ?? null,
|
|
246104
|
-
size
|
|
246299
|
+
size,
|
|
246105
246300
|
updated_at: linkedAt,
|
|
246106
246301
|
blob_ref: blobId,
|
|
246107
246302
|
source_session_id: null,
|
|
@@ -246111,6 +246306,35 @@ function collabDomain(opts) {
|
|
|
246111
246306
|
edit_target: workOrderFileEditTarget(ws, file)
|
|
246112
246307
|
};
|
|
246113
246308
|
return { status: 201, body: { item } };
|
|
246309
|
+
};
|
|
246310
|
+
router.post("/api/workorders/:id/files", async (req) => {
|
|
246311
|
+
const resolved = await resolveUploadTarget(req);
|
|
246312
|
+
if ("denied" in resolved) return resolved.denied;
|
|
246313
|
+
const { target } = resolved;
|
|
246314
|
+
const body2 = req.body ?? {};
|
|
246315
|
+
if (typeof body2.contentBase64 !== "string" || !body2.contentBase64) {
|
|
246316
|
+
return { status: 400, body: { error: { code: "bad_request", message: "\u7F3A\u5C11 contentBase64" } } };
|
|
246317
|
+
}
|
|
246318
|
+
const name = typeof body2.name === "string" && body2.name.trim() ? body2.name.trim() : "untitled";
|
|
246319
|
+
const contentType = typeof body2.contentType === "string" && body2.contentType.trim() ? body2.contentType.trim() : void 0;
|
|
246320
|
+
let bytes2;
|
|
246321
|
+
try {
|
|
246322
|
+
bytes2 = new Uint8Array(Buffer.from(body2.contentBase64, "base64"));
|
|
246323
|
+
} catch (err) {
|
|
246324
|
+
return { status: 400, body: { error: { code: "bad_request", message: `contentBase64 \u89E3\u7801\u5931\u8D25\uFF1A${err instanceof Error ? err.message : String(err)}` } } };
|
|
246325
|
+
}
|
|
246326
|
+
assertFileBytes(bytes2);
|
|
246327
|
+
const blobId = await target.assets.put(bytes2);
|
|
246328
|
+
return linkUploadedFile(req, target, { blobId, name, size: bytes2.byteLength, ...contentType ? { contentType } : {} });
|
|
246329
|
+
});
|
|
246330
|
+
router.post("/api/workorders/:id/files/raw", async (req) => {
|
|
246331
|
+
const resolved = await resolveUploadTarget(req);
|
|
246332
|
+
if ("denied" in resolved) return resolved.denied;
|
|
246333
|
+
const { target } = resolved;
|
|
246334
|
+
const name = rawUploadName(req.raw) ?? "untitled";
|
|
246335
|
+
const contentType = rawUploadContentType(req.raw);
|
|
246336
|
+
const put3 = await putRawUpload(target.assets, req.raw, { maxBytes: currentFileLimits().attachmentBytes });
|
|
246337
|
+
return linkUploadedFile(req, target, { blobId: put3.hash, name, size: put3.size, ...contentType ? { contentType } : {} });
|
|
246114
246338
|
});
|
|
246115
246339
|
router.put("/api/workorders/:id/files/:fileId", async (req) => {
|
|
246116
246340
|
const { kernel, registry: registry2, artifacts } = await resolveCtx(req.auth.companyId);
|
|
@@ -246690,6 +246914,7 @@ var init_collab = __esm({
|
|
|
246690
246914
|
"../server/src/domains/collab/index.ts"() {
|
|
246691
246915
|
"use strict";
|
|
246692
246916
|
init_file_limits2();
|
|
246917
|
+
init_raw_upload();
|
|
246693
246918
|
init_src8();
|
|
246694
246919
|
init_src();
|
|
246695
246920
|
init_chat_workorder_files();
|
|
@@ -263871,6 +264096,44 @@ function readBytesSegments(pool, table, hash3, r5) {
|
|
|
263871
264096
|
cancelled = true;
|
|
263872
264097
|
} };
|
|
263873
264098
|
}
|
|
264099
|
+
async function sweepBlobRows(pool, table, reachable, opts = {}) {
|
|
264100
|
+
const keep = [...reachable];
|
|
264101
|
+
const graceMs = Math.max(0, Math.floor(opts.graceMs ?? 0));
|
|
264102
|
+
const head = await pool.query(
|
|
264103
|
+
`SELECT now() - ($1::bigint * interval '1 millisecond') AS cutoff, (SELECT count(*)::int FROM ${table}) AS total`,
|
|
264104
|
+
[graceMs]
|
|
264105
|
+
);
|
|
264106
|
+
const cutoff = head.rows[0].cutoff;
|
|
264107
|
+
const totalRows = Number(head.rows[0].total);
|
|
264108
|
+
const cand = await pool.query(
|
|
264109
|
+
`SELECT count(*)::int AS n, sum(size)::text AS bytes FROM ${table}
|
|
264110
|
+
WHERE NOT (hash = ANY($1::text[])) AND last_put_at < $2`,
|
|
264111
|
+
[keep, cutoff]
|
|
264112
|
+
);
|
|
264113
|
+
const candidateRows = Number(cand.rows[0]?.n ?? 0);
|
|
264114
|
+
const report = {
|
|
264115
|
+
dryRun: opts.dryRun === true,
|
|
264116
|
+
totalRows,
|
|
264117
|
+
candidateRows,
|
|
264118
|
+
candidateBytes: Number(cand.rows[0]?.bytes ?? 0),
|
|
264119
|
+
deletedRows: 0,
|
|
264120
|
+
deletedBytes: 0,
|
|
264121
|
+
orphansFound: 0,
|
|
264122
|
+
orphansDeleted: 0
|
|
264123
|
+
};
|
|
264124
|
+
if (opts.maxDeleteRatio !== void 0 && totalRows > 0 && candidateRows / totalRows > opts.maxDeleteRatio) {
|
|
264125
|
+
throw new BlobSweepAbortedError(candidateRows, totalRows, opts.maxDeleteRatio);
|
|
264126
|
+
}
|
|
264127
|
+
if (opts.dryRun || candidateRows === 0) return { report, deleted: [], cutoff };
|
|
264128
|
+
const res = await pool.query(
|
|
264129
|
+
`DELETE FROM ${table} WHERE NOT (hash = ANY($1::text[])) AND last_put_at < $2 RETURNING hash, location, size`,
|
|
264130
|
+
[keep, cutoff]
|
|
264131
|
+
);
|
|
264132
|
+
const deleted = res.rows.map((r5) => ({ hash: r5.hash, location: r5.location, size: Number(r5.size) }));
|
|
264133
|
+
report.deletedRows = deleted.length;
|
|
264134
|
+
report.deletedBytes = deleted.reduce((n3, r5) => n3 + r5.size, 0);
|
|
264135
|
+
return { report, deleted, cutoff };
|
|
264136
|
+
}
|
|
263874
264137
|
var import_node_crypto80, ident5, isUniqueViolation, PostgresOplogStore, PG_READ_CHUNK, PostgresBlobStore;
|
|
263875
264138
|
var init_postgres = __esm({
|
|
263876
264139
|
"../storage/src/postgres.ts"() {
|
|
@@ -264093,10 +264356,9 @@ var init_postgres = __esm({
|
|
|
264093
264356
|
const res = await this.pool.query(`SELECT 1 FROM ${this.t} WHERE hash = $1`, [hash3]);
|
|
264094
264357
|
return res.rowCount === 1;
|
|
264095
264358
|
}
|
|
264096
|
-
|
|
264097
|
-
|
|
264098
|
-
|
|
264099
|
-
]);
|
|
264359
|
+
/** 字节都在 bytea,删行即删字节;没有对象后端,`orphans` 不适用。 */
|
|
264360
|
+
async sweep(reachable, opts = {}) {
|
|
264361
|
+
return (await sweepBlobRows(this.pool, this.t, reachable, opts)).report;
|
|
264100
264362
|
}
|
|
264101
264363
|
};
|
|
264102
264364
|
}
|
|
@@ -278963,7 +279225,7 @@ var init_cutover_aware_dispatches = __esm({
|
|
|
278963
279225
|
});
|
|
278964
279226
|
|
|
278965
279227
|
// ../storage/src/cutover-aware-blobs.ts
|
|
278966
|
-
var CutoverAwareBlobStore;
|
|
279228
|
+
var CutoverAwareBlobStore, SharedBlobSweepRefusedError;
|
|
278967
279229
|
var init_cutover_aware_blobs = __esm({
|
|
278968
279230
|
"../storage/src/cutover-aware-blobs.ts"() {
|
|
278969
279231
|
"use strict";
|
|
@@ -279026,11 +279288,20 @@ var init_cutover_aware_blobs = __esm({
|
|
|
279026
279288
|
* ★ 清扫也走**写闸**,而且冻结期必须真的拒——不是"跳过一次"。
|
|
279027
279289
|
*
|
|
279028
279290
|
* 冻结窗口正是 operator 在做最后一次逐行复制的时候:此刻放行 sweep,
|
|
279029
|
-
* 会按一份**还没补完**的可达集去删字节,删掉的是正在搬的那批。
|
|
279291
|
+
* 会按一份**还没补完**的可达集去删字节,删掉的是正在搬的那批。dry-run 也走写闸:冻结期报出来的数没有意义。
|
|
279030
279292
|
*/
|
|
279031
|
-
async sweep(reachable) {
|
|
279032
|
-
|
|
279293
|
+
async sweep(reachable, opts) {
|
|
279294
|
+
const target = await this.pickWritable();
|
|
279295
|
+
if (target === this.o.legacy && !opts?.dryRun) throw new SharedBlobSweepRefusedError(this.o.companyId);
|
|
279296
|
+
return target.sweep(reachable, opts);
|
|
279297
|
+
}
|
|
279298
|
+
};
|
|
279299
|
+
SharedBlobSweepRefusedError = class extends Error {
|
|
279300
|
+
constructor(companyId) {
|
|
279301
|
+
super(`\u516C\u53F8 ${companyId} \u7684 blob \u8FD8\u5728\u5171\u4EAB\u4E8B\u5B9E\u67DC\uFF08\u672A\u5207\u5230\u516C\u53F8 schema\uFF09\uFF0C\u6309\u5355\u516C\u53F8\u53EF\u8FBE\u96C6\u6E05\u626B\u4F1A\u5220\u5230\u522B\u5BB6\u5B57\u8282\uFF1A\u53EA\u5141\u8BB8 dry-run`);
|
|
279302
|
+
this.companyId = companyId;
|
|
279033
279303
|
}
|
|
279304
|
+
name = "SharedBlobSweepRefusedError";
|
|
279034
279305
|
};
|
|
279035
279306
|
}
|
|
279036
279307
|
});
|
|
@@ -279637,6 +279908,319 @@ var init_cutover_aware_trace = __esm({
|
|
|
279637
279908
|
}
|
|
279638
279909
|
});
|
|
279639
279910
|
|
|
279911
|
+
// ../storage/src/externalized-blobs.ts
|
|
279912
|
+
function factObjectKey(namespace, hash3) {
|
|
279913
|
+
return `${factObjectPrefix(namespace)}${hash3.slice(0, 2)}/${hash3}`;
|
|
279914
|
+
}
|
|
279915
|
+
function factObjectPrefix(namespace) {
|
|
279916
|
+
return `facts/${namespace}/`;
|
|
279917
|
+
}
|
|
279918
|
+
async function drainToBuffer(body2) {
|
|
279919
|
+
const parts = [];
|
|
279920
|
+
for await (const chunk of body2) parts.push(chunk);
|
|
279921
|
+
return new Uint8Array(Buffer.concat(parts));
|
|
279922
|
+
}
|
|
279923
|
+
var import_node_crypto87, fs36, BLOB_NAME2, ExternalizedBlobStore, ObjectBlobStore;
|
|
279924
|
+
var init_externalized_blobs = __esm({
|
|
279925
|
+
"../storage/src/externalized-blobs.ts"() {
|
|
279926
|
+
"use strict";
|
|
279927
|
+
import_node_crypto87 = require("node:crypto");
|
|
279928
|
+
fs36 = __toESM(require("node:fs"), 1);
|
|
279929
|
+
init_src();
|
|
279930
|
+
init_blob_object_backend();
|
|
279931
|
+
init_postgres();
|
|
279932
|
+
BLOB_NAME2 = /^[0-9a-f]{64}$/;
|
|
279933
|
+
ExternalizedBlobStore = class _ExternalizedBlobStore {
|
|
279934
|
+
constructor(pool, t, namespace, backend, opts) {
|
|
279935
|
+
this.pool = pool;
|
|
279936
|
+
this.t = t;
|
|
279937
|
+
this.namespace = namespace;
|
|
279938
|
+
this.backend = backend;
|
|
279939
|
+
this.opts = opts;
|
|
279940
|
+
}
|
|
279941
|
+
/** 命名空间 = schema 名(ADR 0638 D3)。 */
|
|
279942
|
+
static async open(pool, schema, backend, opts = {}) {
|
|
279943
|
+
const t = await ensureBlobsTable(pool, schema);
|
|
279944
|
+
return new _ExternalizedBlobStore(pool, t, schema, backend, opts);
|
|
279945
|
+
}
|
|
279946
|
+
key(hash3) {
|
|
279947
|
+
return factObjectKey(this.namespace, hash3);
|
|
279948
|
+
}
|
|
279949
|
+
async put(bytes2) {
|
|
279950
|
+
return (await this.putStream(singleChunk(bytes2))).hash;
|
|
279951
|
+
}
|
|
279952
|
+
async putStream(source, opts = {}) {
|
|
279953
|
+
const staged = await stageBlob(this.backend.stagingDir, source, opts);
|
|
279954
|
+
const contentType = sniffContentType(staged.head);
|
|
279955
|
+
const result = { hash: staged.hash, size: staged.size, ...contentType ? { contentType } : {} };
|
|
279956
|
+
try {
|
|
279957
|
+
await this.place(staged, contentType);
|
|
279958
|
+
} catch (err) {
|
|
279959
|
+
await discardStaged(staged);
|
|
279960
|
+
throw err;
|
|
279961
|
+
}
|
|
279962
|
+
return result;
|
|
279963
|
+
}
|
|
279964
|
+
async place(staged, contentType) {
|
|
279965
|
+
const { hash: hash3, size } = staged;
|
|
279966
|
+
const touched = await this.pool.query(
|
|
279967
|
+
`UPDATE ${this.t} SET last_put_at = now() WHERE hash = $1 RETURNING location`,
|
|
279968
|
+
[hash3]
|
|
279969
|
+
);
|
|
279970
|
+
const row = touched.rows[0];
|
|
279971
|
+
if (row?.location === "pg") {
|
|
279972
|
+
await discardStaged(staged);
|
|
279973
|
+
return;
|
|
279974
|
+
}
|
|
279975
|
+
if (row) {
|
|
279976
|
+
if (await this.backend.head(this.key(hash3)) === null) {
|
|
279977
|
+
await this.backend.commit(staged, this.key(hash3), contentType ? { contentType } : {});
|
|
279978
|
+
} else {
|
|
279979
|
+
await discardStaged(staged);
|
|
279980
|
+
}
|
|
279981
|
+
return;
|
|
279982
|
+
}
|
|
279983
|
+
const bytea = this.opts.dualWrite ? await fs36.promises.readFile(staged.file) : null;
|
|
279984
|
+
await this.backend.commit(staged, this.key(hash3), contentType ? { contentType } : {});
|
|
279985
|
+
await this.pool.query(
|
|
279986
|
+
`INSERT INTO ${this.t} AS b (hash, bytes, size, content_type, location) VALUES ($1, $2, $3, $4, 'ext')
|
|
279987
|
+
ON CONFLICT (hash) DO UPDATE SET last_put_at = now()`,
|
|
279988
|
+
[hash3, bytea, size, contentType ?? null]
|
|
279989
|
+
);
|
|
279990
|
+
}
|
|
279991
|
+
/** §3.2 第 1 条:只读元数据行,不访问对象后端。 */
|
|
279992
|
+
async meta(hash3) {
|
|
279993
|
+
const res = await this.pool.query(`SELECT size, content_type FROM ${this.t} WHERE hash = $1`, [hash3]);
|
|
279994
|
+
const row = res.rows[0];
|
|
279995
|
+
if (!row) return null;
|
|
279996
|
+
return { size: Number(row.size), ...row.content_type ? { contentType: row.content_type } : {} };
|
|
279997
|
+
}
|
|
279998
|
+
async has(hash3) {
|
|
279999
|
+
const res = await this.pool.query(`SELECT 1 FROM ${this.t} WHERE hash = $1`, [hash3]);
|
|
280000
|
+
return res.rowCount === 1;
|
|
280001
|
+
}
|
|
280002
|
+
async get(hash3) {
|
|
280003
|
+
const res = await this.pool.query(
|
|
280004
|
+
`SELECT location, size, CASE WHEN location = 'pg' THEN bytes END AS bytes FROM ${this.t} WHERE hash = $1`,
|
|
280005
|
+
[hash3]
|
|
280006
|
+
);
|
|
280007
|
+
const row = res.rows[0];
|
|
280008
|
+
if (!row) throw new BlobNotFoundError(hash3);
|
|
280009
|
+
if (row.location === "pg") {
|
|
280010
|
+
if (row.bytes === null) throw new BlobBackendError(hash3, `${this.t}.bytes`);
|
|
280011
|
+
return new Uint8Array(row.bytes);
|
|
280012
|
+
}
|
|
280013
|
+
const size = Number(row.size);
|
|
280014
|
+
const handle = await this.openExt(hash3, { start: 0, end: size - 1 });
|
|
280015
|
+
return drainToBuffer(handle.body);
|
|
280016
|
+
}
|
|
280017
|
+
async openRead(hash3, range2) {
|
|
280018
|
+
const res = await this.pool.query(
|
|
280019
|
+
`SELECT location, size, content_type, (location = 'pg' AND bytes IS NULL) AS purged FROM ${this.t} WHERE hash = $1`,
|
|
280020
|
+
[hash3]
|
|
280021
|
+
);
|
|
280022
|
+
const row = res.rows[0];
|
|
280023
|
+
if (!row) throw new BlobNotFoundError(hash3);
|
|
280024
|
+
if (row.purged) throw new BlobBackendError(hash3, `${this.t}.bytes`);
|
|
280025
|
+
const meta = { size: Number(row.size), ...row.content_type ? { contentType: row.content_type } : {} };
|
|
280026
|
+
const r5 = resolveBlobRange(hash3, meta.size, range2);
|
|
280027
|
+
const stream = row.location === "pg" ? readBytesSegments(this.pool, this.t, hash3, r5) : await this.openExt(hash3, r5);
|
|
280028
|
+
return { meta, range: r5, ...stream };
|
|
280029
|
+
}
|
|
280030
|
+
/** §3.2 第 2 条:行在、对象不在 → BlobBackendError;后端其它故障同样包成它(都是 5xx,不是 404)。 */
|
|
280031
|
+
async openExt(hash3, r5) {
|
|
280032
|
+
const key2 = this.key(hash3);
|
|
280033
|
+
try {
|
|
280034
|
+
return await this.backend.openRead(key2, r5);
|
|
280035
|
+
} catch (err) {
|
|
280036
|
+
if (err instanceof ObjectMissingError) throw new BlobBackendError(hash3, key2);
|
|
280037
|
+
throw new BlobBackendError(hash3, key2, err);
|
|
280038
|
+
}
|
|
280039
|
+
}
|
|
280040
|
+
/**
|
|
280041
|
+
* GC 执行(ADR 0638 §3.5):按行清扫(宽限期、删除比例上限、dry-run 见 {@link sweepBlobRows}),
|
|
280042
|
+
* **先删行、再删对象**——崩在中间只留无行对象,读者恒见「没有」,由孤儿扫描清理。
|
|
280043
|
+
* 只动本 schema 的行与本命名空间前缀 `facts/<schema>/` 下的对象:别家同一份字节在别的前缀里,
|
|
280044
|
+
* s3 模式同桶的资产柜在 `assets/` 前缀,都删不到(D3)。
|
|
280045
|
+
*/
|
|
280046
|
+
async sweep(reachable, opts = {}) {
|
|
280047
|
+
const { report, deleted } = await sweepBlobRows(this.pool, this.t, reachable, opts);
|
|
280048
|
+
const ext = deleted.filter((r5) => r5.location === "ext" && BLOB_NAME2.test(r5.hash)).map((r5) => r5.hash);
|
|
280049
|
+
const back = ext.length > 0 ? await this.existingRows(ext) : /* @__PURE__ */ new Set();
|
|
280050
|
+
for (const hash3 of ext) if (!back.has(hash3)) await this.backend.delete(this.key(hash3));
|
|
280051
|
+
if (opts.orphans) {
|
|
280052
|
+
const o3 = await this.sweepOrphans(Math.max(0, opts.graceMs ?? 0), opts.dryRun === true);
|
|
280053
|
+
report.orphansFound = o3.found;
|
|
280054
|
+
report.orphansDeleted = o3.deleted;
|
|
280055
|
+
}
|
|
280056
|
+
return report;
|
|
280057
|
+
}
|
|
280058
|
+
async existingRows(hashes) {
|
|
280059
|
+
const res = await this.pool.query(`SELECT hash FROM ${this.t} WHERE hash = ANY($1::text[])`, [hashes]);
|
|
280060
|
+
return new Set(res.rows.map((r5) => r5.hash));
|
|
280061
|
+
}
|
|
280062
|
+
/**
|
|
280063
|
+
* 孤儿对象扫描(§3.5 第 3 条,`gc --orphans`):列出 `facts/<schema>/` 下全部对象(fs 递归、s3 ListObjectsV2 分页),
|
|
280064
|
+
* 删除**无对应行且早于宽限期**的。宽限期挡住「对象已放、行还没写」的在途写入与租户迁移复制(先放对象、再写行)。
|
|
280065
|
+
* 只认 `facts/<schema>/<hh>/<hash>` 这一种形状,别的文件名一律不碰。
|
|
280066
|
+
*/
|
|
280067
|
+
async sweepOrphans(graceMs, dryRun) {
|
|
280068
|
+
const prefix = factObjectPrefix(this.namespace);
|
|
280069
|
+
const cutoffMs = Date.now() - graceMs;
|
|
280070
|
+
let found2 = 0;
|
|
280071
|
+
let deleted = 0;
|
|
280072
|
+
let page = [];
|
|
280073
|
+
const flush2 = async () => {
|
|
280074
|
+
if (page.length === 0) return;
|
|
280075
|
+
const rows = await this.existingRows(page.map((o3) => o3.hash));
|
|
280076
|
+
for (const o3 of page) {
|
|
280077
|
+
if (rows.has(o3.hash)) continue;
|
|
280078
|
+
found2++;
|
|
280079
|
+
if (!dryRun) {
|
|
280080
|
+
await this.backend.delete(o3.key);
|
|
280081
|
+
deleted++;
|
|
280082
|
+
}
|
|
280083
|
+
}
|
|
280084
|
+
page = [];
|
|
280085
|
+
};
|
|
280086
|
+
for await (const o3 of this.backend.list(prefix)) {
|
|
280087
|
+
const rest = o3.key.slice(prefix.length).split("/");
|
|
280088
|
+
const hash3 = rest[1] ?? "";
|
|
280089
|
+
if (rest.length !== 2 || !BLOB_NAME2.test(hash3) || rest[0] !== hash3.slice(0, 2)) continue;
|
|
280090
|
+
if (o3.mtimeMs >= cutoffMs) continue;
|
|
280091
|
+
page.push({ key: o3.key, hash: hash3 });
|
|
280092
|
+
if (page.length >= 500) await flush2();
|
|
280093
|
+
}
|
|
280094
|
+
await flush2();
|
|
280095
|
+
return { found: found2, deleted };
|
|
280096
|
+
}
|
|
280097
|
+
/** §3.4:仅 s3 + `OASIS_BLOB_S3_DIRECT=redirect` 时给 URL;`pg` 行、不存在的 hash、其它后端一律 null。 */
|
|
280098
|
+
async presignGet(hash3, opts) {
|
|
280099
|
+
if (!this.backend.presignGet) return null;
|
|
280100
|
+
const res = await this.pool.query(`SELECT location, content_type FROM ${this.t} WHERE hash = $1`, [hash3]);
|
|
280101
|
+
const row = res.rows[0];
|
|
280102
|
+
if (!row || row.location !== "ext") return null;
|
|
280103
|
+
const contentType = opts.contentType ?? row.content_type ?? void 0;
|
|
280104
|
+
return this.backend.presignGet(this.key(hash3), { ...opts, ...contentType ? { contentType } : {} });
|
|
280105
|
+
}
|
|
280106
|
+
/* ── 迁移命令(`oasis blobs externalize / internalize`,ADR 0638 §6.2 / §6.3)的单行原语 ── */
|
|
280107
|
+
/**
|
|
280108
|
+
* 把一行 `location='pg'` 搬到对象后端:读 bytea → 原子写 → 读回校验 sha256 → `UPDATE … SET location='ext'`。
|
|
280109
|
+
* **保留 bytea**(清空由 `purgePgBytes` 在观察期后做)。可重复执行:已是 `ext` 的行跳过。
|
|
280110
|
+
* 返回本行字节数;跳过返回 0。
|
|
280111
|
+
*/
|
|
280112
|
+
async externalizeOne(hash3) {
|
|
280113
|
+
const res = await this.pool.query(`SELECT location, bytes, content_type FROM ${this.t} WHERE hash = $1`, [hash3]);
|
|
280114
|
+
const row = res.rows[0];
|
|
280115
|
+
if (!row || row.location !== "pg") return 0;
|
|
280116
|
+
if (row.bytes === null) throw new BlobBackendError(hash3, `${this.t}.bytes`);
|
|
280117
|
+
const bytes2 = new Uint8Array(row.bytes);
|
|
280118
|
+
const staged = await stageBlob(this.backend.stagingDir, singleChunk(bytes2), { expectedHash: hash3 });
|
|
280119
|
+
const contentType = row.content_type ?? sniffContentType(staged.head);
|
|
280120
|
+
await this.backend.commit(staged, this.key(hash3), contentType ? { contentType } : {});
|
|
280121
|
+
const back = await drainToBuffer((await this.backend.openRead(this.key(hash3), { start: 0, end: bytes2.byteLength - 1 })).body);
|
|
280122
|
+
if (!Buffer.from(back).equals(Buffer.from(bytes2))) throw new BlobBackendError(hash3, this.key(hash3), new Error("read-back mismatch"));
|
|
280123
|
+
await this.pool.query(`UPDATE ${this.t} SET location = 'ext' WHERE hash = $1 AND location = 'pg'`, [hash3]);
|
|
280124
|
+
return bytes2.byteLength;
|
|
280125
|
+
}
|
|
280126
|
+
/** 回滚原语:对象写回 bytea、`location='pg'`(§6.3)。已是 `pg` 且 bytea 在的行跳过。 */
|
|
280127
|
+
async internalizeOne(hash3) {
|
|
280128
|
+
const res = await this.pool.query(`SELECT location, size, bytes IS NULL AS empty FROM ${this.t} WHERE hash = $1`, [hash3]);
|
|
280129
|
+
const row = res.rows[0];
|
|
280130
|
+
if (!row || row.location === "pg" && !row.empty) return 0;
|
|
280131
|
+
const size = Number(row.size);
|
|
280132
|
+
const bytes2 = await drainToBuffer((await this.openExt(hash3, { start: 0, end: size - 1 })).body);
|
|
280133
|
+
const actual = (0, import_node_crypto87.createHash)("sha256").update(bytes2).digest("hex");
|
|
280134
|
+
if (actual !== hash3) throw new BlobBackendError(hash3, this.key(hash3), new Error(`sha256 mismatch: ${actual}`));
|
|
280135
|
+
await this.pool.query(`UPDATE ${this.t} SET bytes = $2, location = 'pg' WHERE hash = $1`, [hash3, Buffer.from(bytes2)]);
|
|
280136
|
+
return bytes2.byteLength;
|
|
280137
|
+
}
|
|
280138
|
+
/** 列出待处理的 hash(分批、按 hash 升序,便于中断续跑)。 */
|
|
280139
|
+
async listHashes(where, after, limit) {
|
|
280140
|
+
const cond = where === "pg" ? `location = 'pg'` : where === "ext" ? `location = 'ext'` : `location = 'ext' AND bytes IS NOT NULL`;
|
|
280141
|
+
const res = await this.pool.query(
|
|
280142
|
+
`SELECT hash FROM ${this.t} WHERE ${cond} AND hash > $1 ORDER BY hash LIMIT $2`,
|
|
280143
|
+
[after, limit]
|
|
280144
|
+
);
|
|
280145
|
+
return res.rows.map((r5) => r5.hash);
|
|
280146
|
+
}
|
|
280147
|
+
/** 观察期后清空已外部化行的 bytea(§6.2 第 6 步),一批一条语句。返回本批清掉的行数。 */
|
|
280148
|
+
async purgePgBytes(limit) {
|
|
280149
|
+
const res = await this.pool.query(
|
|
280150
|
+
`UPDATE ${this.t} SET bytes = NULL WHERE hash IN (
|
|
280151
|
+
SELECT hash FROM ${this.t} WHERE location = 'ext' AND bytes IS NOT NULL ORDER BY hash LIMIT $1)`,
|
|
280152
|
+
[limit]
|
|
280153
|
+
);
|
|
280154
|
+
return res.rowCount ?? 0;
|
|
280155
|
+
}
|
|
280156
|
+
/** 各 location 的行数(迁移进度 / 观察期核对 `count(*) WHERE location='pg'` 为 0)。 */
|
|
280157
|
+
async locationCounts() {
|
|
280158
|
+
const res = await this.pool.query(`SELECT
|
|
280159
|
+
count(*) FILTER (WHERE location = 'pg')::int AS pg,
|
|
280160
|
+
count(*) FILTER (WHERE location = 'ext')::int AS ext,
|
|
280161
|
+
count(*) FILTER (WHERE location = 'ext' AND bytes IS NOT NULL)::int AS ext_with_bytes
|
|
280162
|
+
FROM ${this.t}`);
|
|
280163
|
+
const r5 = res.rows[0] ?? {};
|
|
280164
|
+
return { pg: Number(r5.pg ?? 0), ext: Number(r5.ext ?? 0), extWithBytes: Number(r5.ext_with_bytes ?? 0) };
|
|
280165
|
+
}
|
|
280166
|
+
};
|
|
280167
|
+
ObjectBlobStore = class {
|
|
280168
|
+
constructor(backend, prefix) {
|
|
280169
|
+
this.backend = backend;
|
|
280170
|
+
this.prefix = prefix;
|
|
280171
|
+
}
|
|
280172
|
+
key(hash3) {
|
|
280173
|
+
return `${this.prefix.replace(/\/+$/, "")}/${hash3.slice(0, 2)}/${hash3}`;
|
|
280174
|
+
}
|
|
280175
|
+
async put(bytes2) {
|
|
280176
|
+
return (await this.putStream(singleChunk(bytes2))).hash;
|
|
280177
|
+
}
|
|
280178
|
+
async putStream(source, opts = {}) {
|
|
280179
|
+
const staged = await stageBlob(this.backend.stagingDir, source, opts);
|
|
280180
|
+
const contentType = sniffContentType(staged.head);
|
|
280181
|
+
await this.backend.commit(staged, this.key(staged.hash), contentType ? { contentType } : {});
|
|
280182
|
+
return { hash: staged.hash, size: staged.size, ...contentType ? { contentType } : {} };
|
|
280183
|
+
}
|
|
280184
|
+
async meta(hash3) {
|
|
280185
|
+
if (!BLOB_NAME2.test(hash3)) return null;
|
|
280186
|
+
const head = await this.backend.head(this.key(hash3));
|
|
280187
|
+
if (!head) return null;
|
|
280188
|
+
const contentType = head.contentType && head.contentType !== "application/octet-stream" ? head.contentType : void 0;
|
|
280189
|
+
return { size: head.size, ...contentType ? { contentType } : {} };
|
|
280190
|
+
}
|
|
280191
|
+
async has(hash3) {
|
|
280192
|
+
return await this.meta(hash3) !== null;
|
|
280193
|
+
}
|
|
280194
|
+
async get(hash3) {
|
|
280195
|
+
const meta = await this.meta(hash3);
|
|
280196
|
+
if (!meta) throw new BlobNotFoundError(hash3);
|
|
280197
|
+
return drainToBuffer((await this.backend.openRead(this.key(hash3), { start: 0, end: meta.size - 1 })).body);
|
|
280198
|
+
}
|
|
280199
|
+
async openRead(hash3, range2) {
|
|
280200
|
+
const meta = await this.meta(hash3);
|
|
280201
|
+
if (!meta) throw new BlobNotFoundError(hash3);
|
|
280202
|
+
const r5 = resolveBlobRange(hash3, meta.size, range2);
|
|
280203
|
+
try {
|
|
280204
|
+
return { meta, range: r5, ...await this.backend.openRead(this.key(hash3), r5) };
|
|
280205
|
+
} catch (err) {
|
|
280206
|
+
if (err instanceof ObjectMissingError) throw new BlobNotFoundError(hash3);
|
|
280207
|
+
throw err;
|
|
280208
|
+
}
|
|
280209
|
+
}
|
|
280210
|
+
async sweep(reachable) {
|
|
280211
|
+
for await (const o3 of this.backend.list(`${this.prefix.replace(/\/+$/, "")}/`)) {
|
|
280212
|
+
const hash3 = o3.key.slice(o3.key.lastIndexOf("/") + 1);
|
|
280213
|
+
if (BLOB_NAME2.test(hash3) && !reachable.has(hash3)) await this.backend.delete(o3.key);
|
|
280214
|
+
}
|
|
280215
|
+
}
|
|
280216
|
+
async presignGet(hash3, opts) {
|
|
280217
|
+
if (!this.backend.presignGet || !await this.has(hash3)) return null;
|
|
280218
|
+
return this.backend.presignGet(this.key(hash3), opts);
|
|
280219
|
+
}
|
|
280220
|
+
};
|
|
280221
|
+
}
|
|
280222
|
+
});
|
|
280223
|
+
|
|
279640
280224
|
// ../storage/src/tenancy-attribution.ts
|
|
279641
280225
|
function realCompanyId(v2) {
|
|
279642
280226
|
return typeof v2 === "string" && v2.trim().length > 0 ? v2 : null;
|
|
@@ -280820,7 +281404,7 @@ function sourceSetConservation(input) {
|
|
|
280820
281404
|
return input.sourceFinal === input.copied + input.approvedRuntimeConversion + input.approvedArchive + input.approvedDiscard;
|
|
280821
281405
|
}
|
|
280822
281406
|
function payloadChecksum(value2) {
|
|
280823
|
-
return (0,
|
|
281407
|
+
return (0, import_node_crypto88.createHash)("sha256").update(JSON.stringify(value2)).digest("hex");
|
|
280824
281408
|
}
|
|
280825
281409
|
async function copyWorkorderDraftsIdempotent(pool, companyId, companySchema, manifestId, opts) {
|
|
280826
281410
|
const cs = quoteIdent2(companySchema);
|
|
@@ -281002,6 +281586,23 @@ async function copyBlobsIdempotent(pool, companyId, companySchema, manifestId, o
|
|
|
281002
281586
|
const at2 = body2.lastIndexOf("@");
|
|
281003
281587
|
mine.add(at2 >= 0 ? body2.slice(0, at2) : body2);
|
|
281004
281588
|
}
|
|
281589
|
+
if (opts.takeLegacyRows && mine.size > 0) {
|
|
281590
|
+
const legacyNs = opts.legacySchema ?? "public";
|
|
281591
|
+
const hasLocation = (await pool.query(
|
|
281592
|
+
`SELECT 1 FROM information_schema.columns WHERE table_schema=$1 AND table_name='blobs' AND column_name='location'`,
|
|
281593
|
+
[legacyNs]
|
|
281594
|
+
)).rowCount === 1;
|
|
281595
|
+
const ext = hasLocation ? (await pool.query(
|
|
281596
|
+
`SELECT hash FROM ${ls}.blobs WHERE hash = ANY($1::text[]) AND location = 'ext' ORDER BY hash`,
|
|
281597
|
+
[[...mine]]
|
|
281598
|
+
)).rows.map((r5) => r5.hash) : [];
|
|
281599
|
+
if (ext.length > 0 && !opts.objectBackend) {
|
|
281600
|
+
throw new Error(`blob \u590D\u5236\uFF1A${legacyNs}.blobs \u6709 ${ext.length} \u884C\u5B57\u8282\u5728\u5BF9\u8C61\u540E\u7AEF\uFF08location='ext'\uFF09\uFF0C\u9700\u8981\u4F20\u5165 objectBackend \u5148\u590D\u5236\u5BF9\u8C61`);
|
|
281601
|
+
}
|
|
281602
|
+
for (const hash3 of ext) {
|
|
281603
|
+
await opts.objectBackend.copy(factObjectKey(legacyNs, hash3), factObjectKey(companySchema, hash3));
|
|
281604
|
+
}
|
|
281605
|
+
}
|
|
281005
281606
|
let copied = 0;
|
|
281006
281607
|
const client = await pool.connect();
|
|
281007
281608
|
try {
|
|
@@ -281212,13 +281813,14 @@ async function copyTraceFamilyIdempotent(pool, companyId, companySchema, manifes
|
|
|
281212
281813
|
sequences
|
|
281213
281814
|
};
|
|
281214
281815
|
}
|
|
281215
|
-
var
|
|
281816
|
+
var import_node_crypto88, MissingInstallerActorsError, TypeCatalogNotCoveredError, ORG_REGISTRY_TABLES, TRACE_FAMILY;
|
|
281216
281817
|
var init_tenant_migration_operator = __esm({
|
|
281217
281818
|
"../storage/src/tenant-migration-operator.ts"() {
|
|
281218
281819
|
"use strict";
|
|
281219
|
-
|
|
281820
|
+
import_node_crypto88 = require("node:crypto");
|
|
281220
281821
|
init_esm();
|
|
281221
281822
|
init_pg_ident();
|
|
281823
|
+
init_externalized_blobs();
|
|
281222
281824
|
init_tenancy_attribution();
|
|
281223
281825
|
init_tenant_cutover();
|
|
281224
281826
|
MissingInstallerActorsError = class extends Error {
|
|
@@ -281452,8 +282054,8 @@ var init_tenant_new_company_provision = __esm({
|
|
|
281452
282054
|
// ../storage/src/knowledge-snapshot-crypto.ts
|
|
281453
282055
|
function sealSnapshot(snapshot, key2) {
|
|
281454
282056
|
if (snapshot.content.startsWith(PREFIX4)) return structuredClone(snapshot);
|
|
281455
|
-
const iv = (0,
|
|
281456
|
-
const cipher = (0,
|
|
282057
|
+
const iv = (0, import_node_crypto89.randomBytes)(12);
|
|
282058
|
+
const cipher = (0, import_node_crypto89.createCipheriv)("aes-256-gcm", bytes(key2), iv);
|
|
281457
282059
|
cipher.setAAD(aad2(snapshot));
|
|
281458
282060
|
const encrypted = Buffer.concat([cipher.update(snapshot.content, "utf8"), cipher.final()]);
|
|
281459
282061
|
return { ...structuredClone(snapshot), content: `${PREFIX4}${iv.toString("base64url")}.${cipher.getAuthTag().toString("base64url")}.${encrypted.toString("base64url")}` };
|
|
@@ -281463,7 +282065,7 @@ function openSnapshot(snapshot, key2) {
|
|
|
281463
282065
|
const packed = snapshot.content.slice(PREFIX4.length).split(".");
|
|
281464
282066
|
if (packed.length !== 3 || packed.some((part) => !part)) throw new Error(`knowledge snapshot ${snapshot.ref} ciphertext is malformed`);
|
|
281465
282067
|
const [iv, tag, data] = packed;
|
|
281466
|
-
const decipher = (0,
|
|
282068
|
+
const decipher = (0, import_node_crypto89.createDecipheriv)("aes-256-gcm", bytes(key2), Buffer.from(iv, "base64url"));
|
|
281467
282069
|
decipher.setAAD(aad2(snapshot));
|
|
281468
282070
|
decipher.setAuthTag(Buffer.from(tag, "base64url"));
|
|
281469
282071
|
return {
|
|
@@ -281471,11 +282073,11 @@ function openSnapshot(snapshot, key2) {
|
|
|
281471
282073
|
content: Buffer.concat([decipher.update(Buffer.from(data, "base64url")), decipher.final()]).toString("utf8")
|
|
281472
282074
|
};
|
|
281473
282075
|
}
|
|
281474
|
-
var
|
|
282076
|
+
var import_node_crypto89, PREFIX4, bytes, aad2;
|
|
281475
282077
|
var init_knowledge_snapshot_crypto = __esm({
|
|
281476
282078
|
"../storage/src/knowledge-snapshot-crypto.ts"() {
|
|
281477
282079
|
"use strict";
|
|
281478
|
-
|
|
282080
|
+
import_node_crypto89 = require("node:crypto");
|
|
281479
282081
|
PREFIX4 = "oasis-knowledge:v1:";
|
|
281480
282082
|
bytes = (key2) => {
|
|
281481
282083
|
const value2 = Buffer.from(key2);
|
|
@@ -282373,7 +282975,7 @@ function reportedCost(v2) {
|
|
|
282373
282975
|
}
|
|
282374
282976
|
function newUsageBackfillManifestId(now = /* @__PURE__ */ new Date()) {
|
|
282375
282977
|
const stamp = now.toISOString().replace(/[-:]/g, "").replace(/\.\d+Z$/, "Z");
|
|
282376
|
-
return `ulb-${stamp}-${(0,
|
|
282978
|
+
return `ulb-${stamp}-${(0, import_node_crypto90.randomBytes)(3).toString("hex")}`;
|
|
282377
282979
|
}
|
|
282378
282980
|
async function backfillUsageLedger(ledger, opts) {
|
|
282379
282981
|
if (!opts.dryRun && !opts.manifestId) throw new Error("execute \u9700\u8981 manifestId");
|
|
@@ -282922,11 +283524,11 @@ async function audit(c5, s3, manifestId, runId, field, before, after, createdAt)
|
|
|
282922
283524
|
[manifestId, runId, field, before == null ? null : JSON.stringify(before), after == null ? null : JSON.stringify(after), createdAt]
|
|
282923
283525
|
);
|
|
282924
283526
|
}
|
|
282925
|
-
var
|
|
283527
|
+
var import_node_crypto90, BUCKETS, ENDED_SQL, START_EVIDENCE_SQL, CHAT_RUN_SQL, ALLOWED_MODEL_EVIDENCE_TIERS, emptyStep, MODEL_BACKFILL_SOURCE, fourOf, sameUsage, ChatRunLookup;
|
|
282926
283528
|
var init_usage_ledger_backfill = __esm({
|
|
282927
283529
|
"../storage/src/usage-ledger-backfill.ts"() {
|
|
282928
283530
|
"use strict";
|
|
282929
|
-
|
|
283531
|
+
import_node_crypto90 = require("node:crypto");
|
|
282930
283532
|
init_src();
|
|
282931
283533
|
init_pg_ident();
|
|
282932
283534
|
init_postgres_trace();
|
|
@@ -283023,278 +283625,6 @@ var init_usage_ledger_backfill = __esm({
|
|
|
283023
283625
|
}
|
|
283024
283626
|
});
|
|
283025
283627
|
|
|
283026
|
-
// ../storage/src/externalized-blobs.ts
|
|
283027
|
-
function factObjectKey(namespace, hash3) {
|
|
283028
|
-
return `${factObjectPrefix(namespace)}${hash3.slice(0, 2)}/${hash3}`;
|
|
283029
|
-
}
|
|
283030
|
-
function factObjectPrefix(namespace) {
|
|
283031
|
-
return `facts/${namespace}/`;
|
|
283032
|
-
}
|
|
283033
|
-
async function drainToBuffer(body2) {
|
|
283034
|
-
const parts = [];
|
|
283035
|
-
for await (const chunk of body2) parts.push(chunk);
|
|
283036
|
-
return new Uint8Array(Buffer.concat(parts));
|
|
283037
|
-
}
|
|
283038
|
-
var import_node_crypto90, fs36, BLOB_NAME2, ExternalizedBlobStore, ObjectBlobStore;
|
|
283039
|
-
var init_externalized_blobs = __esm({
|
|
283040
|
-
"../storage/src/externalized-blobs.ts"() {
|
|
283041
|
-
"use strict";
|
|
283042
|
-
import_node_crypto90 = require("node:crypto");
|
|
283043
|
-
fs36 = __toESM(require("node:fs"), 1);
|
|
283044
|
-
init_src();
|
|
283045
|
-
init_blob_object_backend();
|
|
283046
|
-
init_postgres();
|
|
283047
|
-
BLOB_NAME2 = /^[0-9a-f]{64}$/;
|
|
283048
|
-
ExternalizedBlobStore = class _ExternalizedBlobStore {
|
|
283049
|
-
constructor(pool, t, namespace, backend, opts) {
|
|
283050
|
-
this.pool = pool;
|
|
283051
|
-
this.t = t;
|
|
283052
|
-
this.namespace = namespace;
|
|
283053
|
-
this.backend = backend;
|
|
283054
|
-
this.opts = opts;
|
|
283055
|
-
}
|
|
283056
|
-
/** 命名空间 = schema 名(ADR 0638 D3)。 */
|
|
283057
|
-
static async open(pool, schema, backend, opts = {}) {
|
|
283058
|
-
const t = await ensureBlobsTable(pool, schema);
|
|
283059
|
-
return new _ExternalizedBlobStore(pool, t, schema, backend, opts);
|
|
283060
|
-
}
|
|
283061
|
-
key(hash3) {
|
|
283062
|
-
return factObjectKey(this.namespace, hash3);
|
|
283063
|
-
}
|
|
283064
|
-
async put(bytes2) {
|
|
283065
|
-
return (await this.putStream(singleChunk(bytes2))).hash;
|
|
283066
|
-
}
|
|
283067
|
-
async putStream(source, opts = {}) {
|
|
283068
|
-
const staged = await stageBlob(this.backend.stagingDir, source, opts);
|
|
283069
|
-
const contentType = sniffContentType(staged.head);
|
|
283070
|
-
const result = { hash: staged.hash, size: staged.size, ...contentType ? { contentType } : {} };
|
|
283071
|
-
try {
|
|
283072
|
-
await this.place(staged, contentType);
|
|
283073
|
-
} catch (err) {
|
|
283074
|
-
await discardStaged(staged);
|
|
283075
|
-
throw err;
|
|
283076
|
-
}
|
|
283077
|
-
return result;
|
|
283078
|
-
}
|
|
283079
|
-
async place(staged, contentType) {
|
|
283080
|
-
const { hash: hash3, size } = staged;
|
|
283081
|
-
const existing = await this.pool.query(`SELECT location FROM ${this.t} WHERE hash = $1`, [hash3]);
|
|
283082
|
-
const row = existing.rows[0];
|
|
283083
|
-
if (row?.location === "pg") {
|
|
283084
|
-
await discardStaged(staged);
|
|
283085
|
-
await this.touch(hash3);
|
|
283086
|
-
return;
|
|
283087
|
-
}
|
|
283088
|
-
if (row) {
|
|
283089
|
-
if (await this.backend.head(this.key(hash3)) === null) {
|
|
283090
|
-
await this.backend.commit(staged, this.key(hash3), contentType ? { contentType } : {});
|
|
283091
|
-
} else {
|
|
283092
|
-
await discardStaged(staged);
|
|
283093
|
-
}
|
|
283094
|
-
await this.touch(hash3);
|
|
283095
|
-
return;
|
|
283096
|
-
}
|
|
283097
|
-
const bytea = this.opts.dualWrite ? await fs36.promises.readFile(staged.file) : null;
|
|
283098
|
-
await this.backend.commit(staged, this.key(hash3), contentType ? { contentType } : {});
|
|
283099
|
-
await this.pool.query(
|
|
283100
|
-
`INSERT INTO ${this.t} AS b (hash, bytes, size, content_type, location) VALUES ($1, $2, $3, $4, 'ext')
|
|
283101
|
-
ON CONFLICT (hash) DO UPDATE SET last_put_at = now()`,
|
|
283102
|
-
[hash3, bytea, size, contentType ?? null]
|
|
283103
|
-
);
|
|
283104
|
-
}
|
|
283105
|
-
async touch(hash3) {
|
|
283106
|
-
await this.pool.query(`UPDATE ${this.t} SET last_put_at = now() WHERE hash = $1`, [hash3]);
|
|
283107
|
-
}
|
|
283108
|
-
/** §3.2 第 1 条:只读元数据行,不访问对象后端。 */
|
|
283109
|
-
async meta(hash3) {
|
|
283110
|
-
const res = await this.pool.query(`SELECT size, content_type FROM ${this.t} WHERE hash = $1`, [hash3]);
|
|
283111
|
-
const row = res.rows[0];
|
|
283112
|
-
if (!row) return null;
|
|
283113
|
-
return { size: Number(row.size), ...row.content_type ? { contentType: row.content_type } : {} };
|
|
283114
|
-
}
|
|
283115
|
-
async has(hash3) {
|
|
283116
|
-
const res = await this.pool.query(`SELECT 1 FROM ${this.t} WHERE hash = $1`, [hash3]);
|
|
283117
|
-
return res.rowCount === 1;
|
|
283118
|
-
}
|
|
283119
|
-
async get(hash3) {
|
|
283120
|
-
const res = await this.pool.query(
|
|
283121
|
-
`SELECT location, size, CASE WHEN location = 'pg' THEN bytes END AS bytes FROM ${this.t} WHERE hash = $1`,
|
|
283122
|
-
[hash3]
|
|
283123
|
-
);
|
|
283124
|
-
const row = res.rows[0];
|
|
283125
|
-
if (!row) throw new BlobNotFoundError(hash3);
|
|
283126
|
-
if (row.location === "pg") {
|
|
283127
|
-
if (row.bytes === null) throw new BlobBackendError(hash3, `${this.t}.bytes`);
|
|
283128
|
-
return new Uint8Array(row.bytes);
|
|
283129
|
-
}
|
|
283130
|
-
const size = Number(row.size);
|
|
283131
|
-
const handle = await this.openExt(hash3, { start: 0, end: size - 1 });
|
|
283132
|
-
return drainToBuffer(handle.body);
|
|
283133
|
-
}
|
|
283134
|
-
async openRead(hash3, range2) {
|
|
283135
|
-
const res = await this.pool.query(
|
|
283136
|
-
`SELECT location, size, content_type, (location = 'pg' AND bytes IS NULL) AS purged FROM ${this.t} WHERE hash = $1`,
|
|
283137
|
-
[hash3]
|
|
283138
|
-
);
|
|
283139
|
-
const row = res.rows[0];
|
|
283140
|
-
if (!row) throw new BlobNotFoundError(hash3);
|
|
283141
|
-
if (row.purged) throw new BlobBackendError(hash3, `${this.t}.bytes`);
|
|
283142
|
-
const meta = { size: Number(row.size), ...row.content_type ? { contentType: row.content_type } : {} };
|
|
283143
|
-
const r5 = resolveBlobRange(hash3, meta.size, range2);
|
|
283144
|
-
const stream = row.location === "pg" ? readBytesSegments(this.pool, this.t, hash3, r5) : await this.openExt(hash3, r5);
|
|
283145
|
-
return { meta, range: r5, ...stream };
|
|
283146
|
-
}
|
|
283147
|
-
/** §3.2 第 2 条:行在、对象不在 → BlobBackendError;后端其它故障同样包成它(都是 5xx,不是 404)。 */
|
|
283148
|
-
async openExt(hash3, r5) {
|
|
283149
|
-
const key2 = this.key(hash3);
|
|
283150
|
-
try {
|
|
283151
|
-
return await this.backend.openRead(key2, r5);
|
|
283152
|
-
} catch (err) {
|
|
283153
|
-
if (err instanceof ObjectMissingError) throw new BlobBackendError(hash3, key2);
|
|
283154
|
-
throw new BlobBackendError(hash3, key2, err);
|
|
283155
|
-
}
|
|
283156
|
-
}
|
|
283157
|
-
/**
|
|
283158
|
-
* GC 执行:删除可达集之外的行,**先删行、再删对象**(崩在中间只留无行对象,读者恒见「没有」)。
|
|
283159
|
-
* 只动本 schema 的行与本命名空间前缀下的对象——别家同一份字节在别的前缀里,删不到(ADR 0638 D3)。
|
|
283160
|
-
* 宽限期、dry-run、删除比例上限、孤儿扫描归 S5(§3.5)。
|
|
283161
|
-
*/
|
|
283162
|
-
async sweep(reachable) {
|
|
283163
|
-
const res = await this.pool.query(
|
|
283164
|
-
`DELETE FROM ${this.t} WHERE NOT (hash = ANY($1::text[])) RETURNING hash, location`,
|
|
283165
|
-
[[...reachable]]
|
|
283166
|
-
);
|
|
283167
|
-
for (const row of res.rows) {
|
|
283168
|
-
if (row.location === "ext" && BLOB_NAME2.test(row.hash)) await this.backend.delete(this.key(row.hash));
|
|
283169
|
-
}
|
|
283170
|
-
}
|
|
283171
|
-
/** §3.4:仅 s3 + `OASIS_BLOB_S3_DIRECT=redirect` 时给 URL;`pg` 行、不存在的 hash、其它后端一律 null。 */
|
|
283172
|
-
async presignGet(hash3, opts) {
|
|
283173
|
-
if (!this.backend.presignGet) return null;
|
|
283174
|
-
const res = await this.pool.query(`SELECT location, content_type FROM ${this.t} WHERE hash = $1`, [hash3]);
|
|
283175
|
-
const row = res.rows[0];
|
|
283176
|
-
if (!row || row.location !== "ext") return null;
|
|
283177
|
-
const contentType = opts.contentType ?? row.content_type ?? void 0;
|
|
283178
|
-
return this.backend.presignGet(this.key(hash3), { ...opts, ...contentType ? { contentType } : {} });
|
|
283179
|
-
}
|
|
283180
|
-
/* ── 迁移命令(`oasis blobs externalize / internalize`,ADR 0638 §6.2 / §6.3)的单行原语 ── */
|
|
283181
|
-
/**
|
|
283182
|
-
* 把一行 `location='pg'` 搬到对象后端:读 bytea → 原子写 → 读回校验 sha256 → `UPDATE … SET location='ext'`。
|
|
283183
|
-
* **保留 bytea**(清空由 `purgePgBytes` 在观察期后做)。可重复执行:已是 `ext` 的行跳过。
|
|
283184
|
-
* 返回本行字节数;跳过返回 0。
|
|
283185
|
-
*/
|
|
283186
|
-
async externalizeOne(hash3) {
|
|
283187
|
-
const res = await this.pool.query(`SELECT location, bytes, content_type FROM ${this.t} WHERE hash = $1`, [hash3]);
|
|
283188
|
-
const row = res.rows[0];
|
|
283189
|
-
if (!row || row.location !== "pg") return 0;
|
|
283190
|
-
if (row.bytes === null) throw new BlobBackendError(hash3, `${this.t}.bytes`);
|
|
283191
|
-
const bytes2 = new Uint8Array(row.bytes);
|
|
283192
|
-
const staged = await stageBlob(this.backend.stagingDir, singleChunk(bytes2), { expectedHash: hash3 });
|
|
283193
|
-
const contentType = row.content_type ?? sniffContentType(staged.head);
|
|
283194
|
-
await this.backend.commit(staged, this.key(hash3), contentType ? { contentType } : {});
|
|
283195
|
-
const back = await drainToBuffer((await this.backend.openRead(this.key(hash3), { start: 0, end: bytes2.byteLength - 1 })).body);
|
|
283196
|
-
if (!Buffer.from(back).equals(Buffer.from(bytes2))) throw new BlobBackendError(hash3, this.key(hash3), new Error("read-back mismatch"));
|
|
283197
|
-
await this.pool.query(`UPDATE ${this.t} SET location = 'ext' WHERE hash = $1 AND location = 'pg'`, [hash3]);
|
|
283198
|
-
return bytes2.byteLength;
|
|
283199
|
-
}
|
|
283200
|
-
/** 回滚原语:对象写回 bytea、`location='pg'`(§6.3)。已是 `pg` 且 bytea 在的行跳过。 */
|
|
283201
|
-
async internalizeOne(hash3) {
|
|
283202
|
-
const res = await this.pool.query(`SELECT location, size, bytes IS NULL AS empty FROM ${this.t} WHERE hash = $1`, [hash3]);
|
|
283203
|
-
const row = res.rows[0];
|
|
283204
|
-
if (!row || row.location === "pg" && !row.empty) return 0;
|
|
283205
|
-
const size = Number(row.size);
|
|
283206
|
-
const bytes2 = await drainToBuffer((await this.openExt(hash3, { start: 0, end: size - 1 })).body);
|
|
283207
|
-
const actual = (0, import_node_crypto90.createHash)("sha256").update(bytes2).digest("hex");
|
|
283208
|
-
if (actual !== hash3) throw new BlobBackendError(hash3, this.key(hash3), new Error(`sha256 mismatch: ${actual}`));
|
|
283209
|
-
await this.pool.query(`UPDATE ${this.t} SET bytes = $2, location = 'pg' WHERE hash = $1`, [hash3, Buffer.from(bytes2)]);
|
|
283210
|
-
return bytes2.byteLength;
|
|
283211
|
-
}
|
|
283212
|
-
/** 列出待处理的 hash(分批、按 hash 升序,便于中断续跑)。 */
|
|
283213
|
-
async listHashes(where, after, limit) {
|
|
283214
|
-
const cond = where === "pg" ? `location = 'pg'` : where === "ext" ? `location = 'ext'` : `location = 'ext' AND bytes IS NOT NULL`;
|
|
283215
|
-
const res = await this.pool.query(
|
|
283216
|
-
`SELECT hash FROM ${this.t} WHERE ${cond} AND hash > $1 ORDER BY hash LIMIT $2`,
|
|
283217
|
-
[after, limit]
|
|
283218
|
-
);
|
|
283219
|
-
return res.rows.map((r5) => r5.hash);
|
|
283220
|
-
}
|
|
283221
|
-
/** 观察期后清空已外部化行的 bytea(§6.2 第 6 步),一批一条语句。返回本批清掉的行数。 */
|
|
283222
|
-
async purgePgBytes(limit) {
|
|
283223
|
-
const res = await this.pool.query(
|
|
283224
|
-
`UPDATE ${this.t} SET bytes = NULL WHERE hash IN (
|
|
283225
|
-
SELECT hash FROM ${this.t} WHERE location = 'ext' AND bytes IS NOT NULL ORDER BY hash LIMIT $1)`,
|
|
283226
|
-
[limit]
|
|
283227
|
-
);
|
|
283228
|
-
return res.rowCount ?? 0;
|
|
283229
|
-
}
|
|
283230
|
-
/** 各 location 的行数(迁移进度 / 观察期核对 `count(*) WHERE location='pg'` 为 0)。 */
|
|
283231
|
-
async locationCounts() {
|
|
283232
|
-
const res = await this.pool.query(`SELECT
|
|
283233
|
-
count(*) FILTER (WHERE location = 'pg')::int AS pg,
|
|
283234
|
-
count(*) FILTER (WHERE location = 'ext')::int AS ext,
|
|
283235
|
-
count(*) FILTER (WHERE location = 'ext' AND bytes IS NOT NULL)::int AS ext_with_bytes
|
|
283236
|
-
FROM ${this.t}`);
|
|
283237
|
-
const r5 = res.rows[0] ?? {};
|
|
283238
|
-
return { pg: Number(r5.pg ?? 0), ext: Number(r5.ext ?? 0), extWithBytes: Number(r5.ext_with_bytes ?? 0) };
|
|
283239
|
-
}
|
|
283240
|
-
};
|
|
283241
|
-
ObjectBlobStore = class {
|
|
283242
|
-
constructor(backend, prefix) {
|
|
283243
|
-
this.backend = backend;
|
|
283244
|
-
this.prefix = prefix;
|
|
283245
|
-
}
|
|
283246
|
-
key(hash3) {
|
|
283247
|
-
return `${this.prefix.replace(/\/+$/, "")}/${hash3.slice(0, 2)}/${hash3}`;
|
|
283248
|
-
}
|
|
283249
|
-
async put(bytes2) {
|
|
283250
|
-
return (await this.putStream(singleChunk(bytes2))).hash;
|
|
283251
|
-
}
|
|
283252
|
-
async putStream(source, opts = {}) {
|
|
283253
|
-
const staged = await stageBlob(this.backend.stagingDir, source, opts);
|
|
283254
|
-
const contentType = sniffContentType(staged.head);
|
|
283255
|
-
await this.backend.commit(staged, this.key(staged.hash), contentType ? { contentType } : {});
|
|
283256
|
-
return { hash: staged.hash, size: staged.size, ...contentType ? { contentType } : {} };
|
|
283257
|
-
}
|
|
283258
|
-
async meta(hash3) {
|
|
283259
|
-
if (!BLOB_NAME2.test(hash3)) return null;
|
|
283260
|
-
const head = await this.backend.head(this.key(hash3));
|
|
283261
|
-
if (!head) return null;
|
|
283262
|
-
const contentType = head.contentType && head.contentType !== "application/octet-stream" ? head.contentType : void 0;
|
|
283263
|
-
return { size: head.size, ...contentType ? { contentType } : {} };
|
|
283264
|
-
}
|
|
283265
|
-
async has(hash3) {
|
|
283266
|
-
return await this.meta(hash3) !== null;
|
|
283267
|
-
}
|
|
283268
|
-
async get(hash3) {
|
|
283269
|
-
const meta = await this.meta(hash3);
|
|
283270
|
-
if (!meta) throw new BlobNotFoundError(hash3);
|
|
283271
|
-
return drainToBuffer((await this.backend.openRead(this.key(hash3), { start: 0, end: meta.size - 1 })).body);
|
|
283272
|
-
}
|
|
283273
|
-
async openRead(hash3, range2) {
|
|
283274
|
-
const meta = await this.meta(hash3);
|
|
283275
|
-
if (!meta) throw new BlobNotFoundError(hash3);
|
|
283276
|
-
const r5 = resolveBlobRange(hash3, meta.size, range2);
|
|
283277
|
-
try {
|
|
283278
|
-
return { meta, range: r5, ...await this.backend.openRead(this.key(hash3), r5) };
|
|
283279
|
-
} catch (err) {
|
|
283280
|
-
if (err instanceof ObjectMissingError) throw new BlobNotFoundError(hash3);
|
|
283281
|
-
throw err;
|
|
283282
|
-
}
|
|
283283
|
-
}
|
|
283284
|
-
async sweep(reachable) {
|
|
283285
|
-
for await (const o3 of this.backend.list(`${this.prefix.replace(/\/+$/, "")}/`)) {
|
|
283286
|
-
const hash3 = o3.key.slice(o3.key.lastIndexOf("/") + 1);
|
|
283287
|
-
if (BLOB_NAME2.test(hash3) && !reachable.has(hash3)) await this.backend.delete(o3.key);
|
|
283288
|
-
}
|
|
283289
|
-
}
|
|
283290
|
-
async presignGet(hash3, opts) {
|
|
283291
|
-
if (!this.backend.presignGet || !await this.has(hash3)) return null;
|
|
283292
|
-
return this.backend.presignGet(this.key(hash3), opts);
|
|
283293
|
-
}
|
|
283294
|
-
};
|
|
283295
|
-
}
|
|
283296
|
-
});
|
|
283297
|
-
|
|
283298
283628
|
// ../../node_modules/.pnpm/@aws-sdk+checksums@3.1001.1/node_modules/@aws-sdk/checksums/dist-es/submodules/flexible-checksums/constants.js
|
|
283299
283629
|
var RequestChecksumCalculation, DEFAULT_REQUEST_CHECKSUM_CALCULATION, ResponseChecksumValidation, DEFAULT_RESPONSE_CHECKSUM_VALIDATION, ChecksumAlgorithm, ChecksumLocation, DEFAULT_CHECKSUM_ALGORITHM;
|
|
283300
283630
|
var init_constants = __esm({
|
|
@@ -320211,6 +320541,7 @@ __export(src_exports, {
|
|
|
320211
320541
|
S3_DEFAULT_TIMEOUTS: () => S3_DEFAULT_TIMEOUTS,
|
|
320212
320542
|
STALE_STAGING_MS: () => STALE_STAGING_MS,
|
|
320213
320543
|
SchemaMappingConflictError: () => SchemaMappingConflictError,
|
|
320544
|
+
SharedBlobSweepRefusedError: () => SharedBlobSweepRefusedError,
|
|
320214
320545
|
SkillCatalogRetiredError: () => SkillCatalogRetiredError,
|
|
320215
320546
|
TENANCY_MOVE_PLANS: () => TENANCY_MOVE_PLANS,
|
|
320216
320547
|
TENANCY_TABLES: () => TENANCY_TABLES,
|
|
@@ -320311,6 +320642,7 @@ __export(src_exports, {
|
|
|
320311
320642
|
singleChunk: () => singleChunk,
|
|
320312
320643
|
sourceSetConservation: () => sourceSetConservation,
|
|
320313
320644
|
stageBlob: () => stageBlob,
|
|
320645
|
+
sweepBlobRows: () => sweepBlobRows,
|
|
320314
320646
|
synthesizedItemId: () => synthesizedItemId,
|
|
320315
320647
|
undoUsageLedgerBackfill: () => undoUsageLedgerBackfill,
|
|
320316
320648
|
usageBucket: () => usageBucket,
|
|
@@ -337630,9 +337962,15 @@ var COMMAND_DECLS = {
|
|
|
337630
337962
|
// —— 运维 ——
|
|
337631
337963
|
gc: {
|
|
337632
337964
|
group: "\u8FD0\u7EF4",
|
|
337633
|
-
usage: "oasis gc",
|
|
337634
|
-
description: "\u5B64\u513F blob \u56DE\u6536\uFF08mark-and-sweep\uFF1B\u64CD\u4F5C\u8005\u547D\u4EE4\
|
|
337635
|
-
|
|
337965
|
+
usage: "oasis gc [--dry-run] [--grace-hours 24] [--max-delete-ratio 0.2] [--orphans]",
|
|
337966
|
+
description: "\u4E8B\u5B9E\u67DC\u5B64\u513F blob \u56DE\u6536\uFF08mark-and-sweep\uFF1B\u64CD\u4F5C\u8005\u547D\u4EE4\uFF0C\u624B\u52A8\u89E6\u53D1\uFF0CADR 0638 \xA73.5\uFF09\u3002\u53EF\u8FBE\u96C6 = \u4EA4\u4ED8\u7269 + \u9879\u76EE\u6587\u4EF6\uFF1B\u6700\u540E\u4E00\u6B21\u5199\u5165\u5728\u5BBD\u9650\u671F\u5185\u7684\u4E0D\u5220\uFF1B\u5F85\u5220\u8D85\u8FC7\u6BD4\u4F8B\u4E0A\u9650\u5219\u4E2D\u6B62\u3001\u4E00\u6761\u4E0D\u5220\u3002\u5148\u8DD1 --dry-run \u770B\u62A5\u544A\u518D\u771F\u5220\u3002",
|
|
337967
|
+
flags: [
|
|
337968
|
+
{ name: "dry-run", desc: "\u53EA\u62A5\u4F1A\u5220\u591A\u5C11\u884C / \u5B57\u8282 / \u5B64\u513F\u5BF9\u8C61\uFF0C\u4E00\u6761\u4E0D\u5220", boolean: true },
|
|
337969
|
+
{ name: "grace-hours", desc: "\u5BBD\u9650\u671F\uFF08\u5C0F\u65F6\uFF0C\u7F3A\u7701 24\uFF09\uFF1A\u8FD9\u6BB5\u65F6\u95F4\u5185\u5199\u5165\u8FC7\u7684 blob \u4E0D\u5220" },
|
|
337970
|
+
{ name: "max-delete-ratio", desc: "\u5F85\u5220\u884C\u6570 / \u603B\u884C\u6570\u4E0A\u9650\uFF080\u20131\uFF0C\u7F3A\u7701 0.2\uFF09\uFF1B\u8D85\u8FC7\u5373\u4E2D\u6B62" },
|
|
337971
|
+
{ name: "orphans", desc: "\u540C\u65F6\u6E05\u626B\u5BF9\u8C61\u5B58\u50A8\u91CC\u65E0\u5143\u6570\u636E\u884C\u7684\u5B64\u513F\u5BF9\u8C61\uFF08fs / s3 \u540E\u7AEF\uFF09", boolean: true }
|
|
337972
|
+
],
|
|
337973
|
+
examples: ["oasis gc --dry-run", "oasis gc --dry-run --orphans", "oasis gc --orphans"]
|
|
337636
337974
|
},
|
|
337637
337975
|
broadcasts: {
|
|
337638
337976
|
group: "\u8FD0\u7EF4",
|
|
@@ -339277,7 +339615,7 @@ async function waitForCommandAuthorization(base, headers, draftId, fallback2) {
|
|
|
339277
339615
|
}
|
|
339278
339616
|
}
|
|
339279
339617
|
}
|
|
339280
|
-
var CLI_VERSION = "2.2.
|
|
339618
|
+
var CLI_VERSION = "2.2.119" ? "2.2.119" : "dev";
|
|
339281
339619
|
function apiHeaders(token2, company) {
|
|
339282
339620
|
const headers = {
|
|
339283
339621
|
authorization: `Bearer ${token2}`,
|
|
@@ -342301,7 +342639,14 @@ ${res.warning}`);
|
|
|
342301
342639
|
break;
|
|
342302
342640
|
}
|
|
342303
342641
|
case "gc": {
|
|
342304
|
-
const
|
|
342642
|
+
const graceHours = flags.get("grace-hours");
|
|
342643
|
+
const maxDeleteRatio = flags.get("max-delete-ratio");
|
|
342644
|
+
const { message } = await api.cmd("gc", {
|
|
342645
|
+
...flags.get("dry-run") === "true" ? { dryRun: true } : {},
|
|
342646
|
+
...flags.get("orphans") === "true" ? { orphans: true } : {},
|
|
342647
|
+
...graceHours !== void 0 ? { graceHours: Number(graceHours) } : {},
|
|
342648
|
+
...maxDeleteRatio !== void 0 ? { maxDeleteRatio: Number(maxDeleteRatio) } : {}
|
|
342649
|
+
});
|
|
342305
342650
|
println(message);
|
|
342306
342651
|
break;
|
|
342307
342652
|
}
|
|
@@ -343448,7 +343793,7 @@ function shimScript() {
|
|
|
343448
343793
|
}
|
|
343449
343794
|
|
|
343450
343795
|
// src/index.ts
|
|
343451
|
-
var PKG_VERSION = true ? "2.2.
|
|
343796
|
+
var PKG_VERSION = true ? "2.2.119" : "dev";
|
|
343452
343797
|
var LOCAL_BIN = localBin();
|
|
343453
343798
|
var NPM_PREFIX = npmPrefix();
|
|
343454
343799
|
var INSTANCE = DEFAULT_INSTANCE;
|