@opendikw/web-client 0.1.1 → 0.1.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +30 -0
- package/dist/convert/mineru-convert.js +6 -13
- package/dist/core/client.js +38 -2
- package/dist/core/import-safety.d.ts +15 -0
- package/dist/core/import-safety.js +118 -0
- package/dist/import/import-bundle.js +4 -1
- package/dist/import/kebab-source-name.js +11 -1
- package/dist/import/tar-reader.d.ts +5 -0
- package/dist/import/tar-reader.js +24 -22
- package/dist/import/tar-stream.d.ts +3 -0
- package/dist/import/tar-stream.js +95 -0
- package/dist/import/tar.js +1 -1
- package/dist/types/index.d.ts +2 -0
- package/package.json +1 -1
package/README.md
CHANGED
|
@@ -22,6 +22,36 @@ Conversion and translation caches accept `{ namespace }`. Omit it to retain
|
|
|
22
22
|
the existing cache database names and TTL. Applications may use an identity
|
|
23
23
|
and Core-specific namespace to keep local caches separate.
|
|
24
24
|
|
|
25
|
+
`convertSource(file)` normalizes long upload names itself and preserves the true
|
|
26
|
+
name in provenance. `DikwClient.importBundle` checks active/inactive indexed source
|
|
27
|
+
paths before posting. Packages that would replace a source or an active source's
|
|
28
|
+
referenced attachment are returned in `rejected` with `source_path_exists` or
|
|
29
|
+
`source_asset_exists`; safe packages keep their original manifest ids. A wholly
|
|
30
|
+
rejected preflight makes no import POST and returns empty `import_id`/`applied_at`,
|
|
31
|
+
zero uploaded bytes and `committed: []`. Always inspect `rejected` and optional
|
|
32
|
+
`warnings` (including `source_content_matches`); matching bodies may have different
|
|
33
|
+
metadata/assets. Skip ingest/synth when `committed` is empty.
|
|
34
|
+
|
|
35
|
+
If an active source uses a local absolute attachment reference, the HTTP client
|
|
36
|
+
cannot determine its filesystem target. It conservatively rejects incoming
|
|
37
|
+
packages with attachments as `source_asset_scope_unknown`, with the existing
|
|
38
|
+
source path and original reference in `detail`; Markdown-only packages can still
|
|
39
|
+
commit. Change that source's attachment reference to a relative path before
|
|
40
|
+
retrying. This may reject an unrelated attachment rather than risk replacing data.
|
|
41
|
+
|
|
42
|
+
This preflight uses Core's index. It cannot reserve paths atomically or see
|
|
43
|
+
unindexed files; inactive source names are protected, but their bodies cannot be
|
|
44
|
+
read to discover attachment references. Core remains responsible for concurrent
|
|
45
|
+
write protection. Optional warning inspection preserves non-conflicting Core tar
|
|
46
|
+
formats unchanged; if a conflicting archive cannot be safely filtered by the
|
|
47
|
+
shared USTAR reader, the remainder is rejected as `unsupported_safe_import_archive`.
|
|
48
|
+
|
|
49
|
+
Preflight validates USTAR incrementally and only collects the current Markdown
|
|
50
|
+
body for the optional duplicate warning. It streams retained header/data blocks
|
|
51
|
+
directly into gzip when filtering, without collecting or copying attachment
|
|
52
|
+
contents into a whole decompressed archive. Non-conflicting payloads keep their
|
|
53
|
+
original bytes; filtering collects only its compressed result for the POST.
|
|
54
|
+
|
|
25
55
|
Build explicitly with `npm run build:packages` from the repository root;
|
|
26
56
|
install scripts are disabled. npm packages contain compiled ESM and declarations,
|
|
27
57
|
this README and the MIT license. Import through `exports`, not package source paths.
|
|
@@ -10,7 +10,7 @@
|
|
|
10
10
|
// Idempotency layer (b) in the plan: IndexedDB keyed by sha256(input). Same
|
|
11
11
|
// file content → same cache key, regardless of filename. mineruVersion gates
|
|
12
12
|
// invalidation when the rewrite logic changes.
|
|
13
|
-
import { sha256Hex } from "../import/import-bundle.js";
|
|
13
|
+
import { lowerExt, sha256Hex } from "../import/import-bundle.js";
|
|
14
14
|
import { kebabStem } from "../import/kebab-source-name.js";
|
|
15
15
|
import { readTar, TarReaderError } from "../import/tar-reader.js";
|
|
16
16
|
export const MINERU_EXTENSIONS = new Set([
|
|
@@ -71,11 +71,13 @@ export async function convertSource(file, opts = {}) {
|
|
|
71
71
|
}
|
|
72
72
|
}
|
|
73
73
|
opts.onProgress?.({ phase: "uploading" });
|
|
74
|
+
const stem = kebabStem(file.name);
|
|
75
|
+
const uploadName = `${stem}${lowerExt(file.name)}`;
|
|
74
76
|
const fd = new FormData();
|
|
75
|
-
fd.append("file", file);
|
|
77
|
+
fd.append("file", file, uploadName);
|
|
76
78
|
let submitUrl = `/web/mineru/convert?inputSha=${encodeURIComponent(inputSha)}`;
|
|
77
|
-
if (opts.originalFilename) {
|
|
78
|
-
submitUrl += `&originalFilename=${encodeURIComponent(opts.originalFilename)}`;
|
|
79
|
+
if (opts.originalFilename || file.name !== uploadName) {
|
|
80
|
+
submitUrl += `&originalFilename=${encodeURIComponent(opts.originalFilename || file.name)}`;
|
|
79
81
|
}
|
|
80
82
|
// Submit returns a job id immediately; the conversion then runs detached on
|
|
81
83
|
// the sidecar. Polling and the result fetch are each short, independent
|
|
@@ -112,7 +114,6 @@ export async function convertSource(file, opts = {}) {
|
|
|
112
114
|
}
|
|
113
115
|
throw err;
|
|
114
116
|
}
|
|
115
|
-
const stem = stemOf(file.name);
|
|
116
117
|
const mdName = `${stem}.md`;
|
|
117
118
|
let markdown = null;
|
|
118
119
|
const assets = new Map();
|
|
@@ -176,14 +177,6 @@ function syntheticFile(name, webkitRelativePath, data, type) {
|
|
|
176
177
|
});
|
|
177
178
|
return file;
|
|
178
179
|
}
|
|
179
|
-
function stemOf(fileName) {
|
|
180
|
-
const base = fileName.replace(/^.*[\\/]/, "");
|
|
181
|
-
const i = base.lastIndexOf(".");
|
|
182
|
-
const stem = i < 0 ? base : base.slice(0, i);
|
|
183
|
-
// Strip path / wikilink-breaking chars so the synthesized archive path
|
|
184
|
-
// doesn't accidentally split into multiple segments.
|
|
185
|
-
return stem.replace(/[\\/]/g, "_").replace(/[\]|]/g, "_") || "untitled";
|
|
186
|
-
}
|
|
187
180
|
function signalThrowIfAborted(signal) {
|
|
188
181
|
if (signal?.aborted) {
|
|
189
182
|
throw new MineruConvertError("aborted", "aborted");
|
package/dist/core/client.js
CHANGED
|
@@ -1,4 +1,5 @@
|
|
|
1
1
|
import { decodeNdjsonStream } from "./ndjson.js";
|
|
2
|
+
import { prepareImport, protectReferencedAssets } from "./import-safety.js";
|
|
2
3
|
export class DikwClientError extends Error {
|
|
3
4
|
status;
|
|
4
5
|
code;
|
|
@@ -118,12 +119,47 @@ export class DikwClient {
|
|
|
118
119
|
signal,
|
|
119
120
|
});
|
|
120
121
|
}
|
|
121
|
-
importBundle(payload, manifestJson, signal) {
|
|
122
|
+
async importBundle(payload, manifestJson, signal) {
|
|
123
|
+
const manifest = JSON.parse(manifestJson);
|
|
124
|
+
let rejected = [];
|
|
125
|
+
let warnings = [];
|
|
126
|
+
if (manifest.packages?.length) {
|
|
127
|
+
const groups = await Promise.all([true, false].map((active) => this.get("/v1/base/pages", {
|
|
128
|
+
params: { layer: "source", active },
|
|
129
|
+
signal,
|
|
130
|
+
})));
|
|
131
|
+
const pages = Array.from(new Map(groups.flat().map((page) => [page.path, page])).values());
|
|
132
|
+
const incomingAssets = new Set(manifest.packages.flatMap((pkg) => pkg.asset_paths));
|
|
133
|
+
const protectedAssets = new Map();
|
|
134
|
+
if (incomingAssets.size) {
|
|
135
|
+
const readable = pages.filter((page) => page.active !== false);
|
|
136
|
+
for (let i = 0; i < readable.length; i += 6) {
|
|
137
|
+
const bodies = await Promise.all(readable
|
|
138
|
+
.slice(i, i + 6)
|
|
139
|
+
.map((page) => this.get(`/v1/base/pages/${page.path.split("/").map(encodeURIComponent).join("/")}`, { signal })));
|
|
140
|
+
for (const body of bodies)
|
|
141
|
+
protectReferencedAssets(body, incomingAssets, protectedAssets);
|
|
142
|
+
}
|
|
143
|
+
}
|
|
144
|
+
const prepared = await prepareImport(payload, manifest, pages, protectedAssets);
|
|
145
|
+
signal?.throwIfAborted();
|
|
146
|
+
rejected = prepared.rejected;
|
|
147
|
+
warnings = prepared.warnings;
|
|
148
|
+
if (!prepared.payload)
|
|
149
|
+
return { import_id: "", applied_at: "", files_count: 0, bytes: 0, committed: [], rejected };
|
|
150
|
+
payload = prepared.payload;
|
|
151
|
+
manifestJson = JSON.stringify(prepared.manifest);
|
|
152
|
+
}
|
|
122
153
|
const form = new FormData();
|
|
123
154
|
// Field names match dikw-core's routes_import.py multipart contract.
|
|
124
155
|
form.append("payload", payload, "import.tar.gz");
|
|
125
156
|
form.append("manifest", manifestJson);
|
|
126
|
-
|
|
157
|
+
const result = await this.postMultipart("/v1/import", form, signal);
|
|
158
|
+
if (rejected.length)
|
|
159
|
+
result.rejected = [...rejected, ...result.rejected];
|
|
160
|
+
if (warnings.length)
|
|
161
|
+
result.warnings = [...warnings, ...(result.warnings ?? [])];
|
|
162
|
+
return result;
|
|
127
163
|
}
|
|
128
164
|
startIngest(opts = {}, signal) {
|
|
129
165
|
return this.post("/v1/ingest", { no_embed: opts.noEmbed ?? false }, { signal });
|
|
@@ -0,0 +1,15 @@
|
|
|
1
|
+
import type { DocumentRecord, PageReadResult, RejectedPackage } from "../types/index.js";
|
|
2
|
+
import { type ManifestJson } from "../import/import-bundle.js";
|
|
3
|
+
export interface AssetProtection {
|
|
4
|
+
existing_path?: string;
|
|
5
|
+
original_ref?: string;
|
|
6
|
+
}
|
|
7
|
+
export declare function protectReferencedAssets(page: PageReadResult, incoming: Set<string>, protectedPaths: Map<string, AssetProtection>): void;
|
|
8
|
+
/** Core 0.6.9 replaces matching paths. Reject indexed source collisions before
|
|
9
|
+
* submitting; this is a preflight, not an atomic server-side write guard. */
|
|
10
|
+
export declare function prepareImport(payload: Blob, manifest: ManifestJson, pages: DocumentRecord[], protectedAssets: Map<string, AssetProtection>): Promise<{
|
|
11
|
+
payload: Blob | null;
|
|
12
|
+
manifest: ManifestJson;
|
|
13
|
+
rejected: RejectedPackage[];
|
|
14
|
+
warnings: RejectedPackage[];
|
|
15
|
+
}>;
|
|
@@ -0,0 +1,118 @@
|
|
|
1
|
+
import { sha256HexString } from "../import/import-bundle.js";
|
|
2
|
+
import { extractAssetRefs, isRemoteRef, posixJoinNormalize, stripFrontmatter, } from "../import/md-asset-refs.js";
|
|
3
|
+
import { streamTar } from "../import/tar-stream.js";
|
|
4
|
+
export function protectReferencedAssets(page, incoming, protectedPaths) {
|
|
5
|
+
for (const ref of extractAssetRefs(page.body)) {
|
|
6
|
+
if (isRemoteRef(ref.originalPath))
|
|
7
|
+
continue;
|
|
8
|
+
if (ref.originalPath.startsWith("/")) {
|
|
9
|
+
// Core resolves local absolute paths against its filesystem, which HTTP
|
|
10
|
+
// consumers cannot inspect. Any incoming asset might replace that target.
|
|
11
|
+
for (const path of incoming)
|
|
12
|
+
protectedPaths.set(path, { existing_path: page.path, original_ref: ref.originalPath });
|
|
13
|
+
continue;
|
|
14
|
+
}
|
|
15
|
+
const mdDir = page.path.slice(0, Math.max(0, page.path.lastIndexOf("/")));
|
|
16
|
+
// The new archive cannot tell us which existing Core candidate was used.
|
|
17
|
+
// Protect every incoming sibling/root candidate, including sources/ prefixes.
|
|
18
|
+
for (const base of [mdDir, "", "sources"]) {
|
|
19
|
+
const candidate = posixJoinNormalize(base, ref.originalPath);
|
|
20
|
+
if (!candidate.startsWith("..") && incoming.has(candidate) && !protectedPaths.has(candidate))
|
|
21
|
+
protectedPaths.set(candidate, {});
|
|
22
|
+
}
|
|
23
|
+
}
|
|
24
|
+
}
|
|
25
|
+
/** Core 0.6.9 replaces matching paths. Reject indexed source collisions before
|
|
26
|
+
* submitting; this is a preflight, not an atomic server-side write guard. */
|
|
27
|
+
export async function prepareImport(payload, manifest, pages, protectedAssets) {
|
|
28
|
+
const paths = new Set(pages.map((page) => page.path));
|
|
29
|
+
const rejected = [];
|
|
30
|
+
const warnings = [];
|
|
31
|
+
const packages = manifest.packages.filter((pkg) => {
|
|
32
|
+
if (paths.has(pkg.md_path)) {
|
|
33
|
+
rejected.push({
|
|
34
|
+
id: pkg.id,
|
|
35
|
+
code: "source_path_exists",
|
|
36
|
+
detail: { md_path: pkg.md_path, existing_path: pkg.md_path },
|
|
37
|
+
});
|
|
38
|
+
return false;
|
|
39
|
+
}
|
|
40
|
+
const asset = pkg.asset_paths.find((path) => protectedAssets.has(path) || paths.has(path));
|
|
41
|
+
if (asset) {
|
|
42
|
+
const protection = protectedAssets.get(asset);
|
|
43
|
+
rejected.push({
|
|
44
|
+
id: pkg.id,
|
|
45
|
+
code: protection?.original_ref ? "source_asset_scope_unknown" : "source_asset_exists",
|
|
46
|
+
detail: { md_path: pkg.md_path, asset_path: asset, ...protection },
|
|
47
|
+
});
|
|
48
|
+
return false;
|
|
49
|
+
}
|
|
50
|
+
return true;
|
|
51
|
+
});
|
|
52
|
+
if (pages.length === 0)
|
|
53
|
+
return { payload, manifest, rejected, warnings };
|
|
54
|
+
const retained = new Set(packages.flatMap((pkg) => [pkg.md_path, ...pkg.asset_paths]));
|
|
55
|
+
const files = manifest.files.filter((file) => retained.has(file.path));
|
|
56
|
+
const filtered = {
|
|
57
|
+
files,
|
|
58
|
+
packages,
|
|
59
|
+
total_bytes: files.reduce((sum, file) => sum + file.size, 0),
|
|
60
|
+
};
|
|
61
|
+
if (packages.length === 0)
|
|
62
|
+
return { payload: null, manifest: filtered, rejected, warnings };
|
|
63
|
+
const byPath = new Map(packages.map((pkg) => [pkg.md_path, pkg]));
|
|
64
|
+
const byHash = new Map(pages.map((page) => [page.hash, page.path]));
|
|
65
|
+
const matches = new Map();
|
|
66
|
+
let compressed;
|
|
67
|
+
try {
|
|
68
|
+
const tar = streamTar(payload.stream().pipeThrough(new DecompressionStream("gzip")), (path) => rejected.length > 0 && retained.has(path), (path) => byPath.has(path), async (path, text) => {
|
|
69
|
+
const body = stripFrontmatter(text).replace(/\r\n/g, "\n").trim();
|
|
70
|
+
matches.set(path, byHash.get(await sha256HexString(body)));
|
|
71
|
+
});
|
|
72
|
+
if (rejected.length > 0)
|
|
73
|
+
compressed = await new Response(tar.pipeThrough(new CompressionStream("gzip"))).blob();
|
|
74
|
+
else {
|
|
75
|
+
const reader = tar.getReader();
|
|
76
|
+
try {
|
|
77
|
+
while (!(await reader.read()).done) {
|
|
78
|
+
/* inspect without collecting asset data */
|
|
79
|
+
}
|
|
80
|
+
}
|
|
81
|
+
finally {
|
|
82
|
+
reader.releaseLock();
|
|
83
|
+
}
|
|
84
|
+
}
|
|
85
|
+
}
|
|
86
|
+
catch (err) {
|
|
87
|
+
// Warning inspection is optional; Core accepts more tar variants than our
|
|
88
|
+
// conversion reader. Forward unchanged when no protected path would be written.
|
|
89
|
+
if (rejected.length === 0)
|
|
90
|
+
return { payload, manifest, rejected, warnings };
|
|
91
|
+
// Filtering an unsupported tar cannot safely preserve Core's semantics.
|
|
92
|
+
// Reject the remainder rather than accidentally forward the conflicting files.
|
|
93
|
+
for (const pkg of packages)
|
|
94
|
+
rejected.push({
|
|
95
|
+
id: pkg.id,
|
|
96
|
+
code: "unsupported_safe_import_archive",
|
|
97
|
+
detail: { md_path: pkg.md_path, message: err instanceof Error ? err.message : String(err) },
|
|
98
|
+
});
|
|
99
|
+
return {
|
|
100
|
+
payload: null,
|
|
101
|
+
manifest: { files: [], packages: [], total_bytes: 0 },
|
|
102
|
+
rejected,
|
|
103
|
+
warnings,
|
|
104
|
+
};
|
|
105
|
+
}
|
|
106
|
+
for (const pkg of packages) {
|
|
107
|
+
const existing = matches.get(pkg.md_path);
|
|
108
|
+
if (existing)
|
|
109
|
+
warnings.push({
|
|
110
|
+
id: pkg.id,
|
|
111
|
+
code: "source_content_matches",
|
|
112
|
+
detail: { md_path: pkg.md_path, existing_path: existing },
|
|
113
|
+
});
|
|
114
|
+
}
|
|
115
|
+
if (rejected.length === 0)
|
|
116
|
+
return { payload, manifest, rejected, warnings };
|
|
117
|
+
return { payload: compressed, manifest: filtered, rejected, warnings };
|
|
118
|
+
}
|
|
@@ -86,7 +86,10 @@ export function scanFiles(files) {
|
|
|
86
86
|
function appendSuffix(stem, n) {
|
|
87
87
|
const suffix = `-${n}`;
|
|
88
88
|
const max = 28 - suffix.length;
|
|
89
|
-
const
|
|
89
|
+
const points = Array.from(stem).slice(0, max);
|
|
90
|
+
while (new TextEncoder().encode(points.join("") + suffix).length > 97)
|
|
91
|
+
points.pop();
|
|
92
|
+
const capped = points.join("").replace(/-+$/g, "");
|
|
90
93
|
return `${capped}${suffix}`;
|
|
91
94
|
}
|
|
92
95
|
/** Normalize a scanned import: rename each markdown to a Unicode-kebab basename,
|
|
@@ -22,7 +22,17 @@ export function kebabStem(name, maxStem = MAX_STEM) {
|
|
|
22
22
|
.replace(/^-+|-+$/g, "");
|
|
23
23
|
// Cap by code point (``Array.from`` never splits a surrogate pair), then
|
|
24
24
|
// re-trim any trailing hyphen the cut exposed.
|
|
25
|
-
|
|
25
|
+
let capped = "";
|
|
26
|
+
let bytes = 0;
|
|
27
|
+
for (const point of Array.from(kebab).slice(0, maxStem)) {
|
|
28
|
+
const size = new TextEncoder().encode(point).length;
|
|
29
|
+
// Reserve three bytes for .md in the USTAR name field.
|
|
30
|
+
if (bytes + size > 97)
|
|
31
|
+
break;
|
|
32
|
+
capped += point;
|
|
33
|
+
bytes += size;
|
|
34
|
+
}
|
|
35
|
+
capped = capped.replace(/-+$/g, "");
|
|
26
36
|
return capped || "untitled";
|
|
27
37
|
}
|
|
28
38
|
/** Kebab-case a source filename and force the ``.md`` extension. */
|
|
@@ -6,4 +6,9 @@ export interface TarEntry {
|
|
|
6
6
|
archivePath: string;
|
|
7
7
|
data: Uint8Array;
|
|
8
8
|
}
|
|
9
|
+
/** Shared header validation for buffered conversion and streamed import checks. */
|
|
10
|
+
export declare function readTarHeader(input: Uint8Array, pos?: number): {
|
|
11
|
+
archivePath: string;
|
|
12
|
+
size: number;
|
|
13
|
+
} | null;
|
|
9
14
|
export declare function readTar(input: Uint8Array): TarEntry[];
|
|
@@ -71,37 +71,39 @@ function isUnsafePath(path) {
|
|
|
71
71
|
}
|
|
72
72
|
return false;
|
|
73
73
|
}
|
|
74
|
+
/** Shared header validation for buffered conversion and streamed import checks. */
|
|
75
|
+
export function readTarHeader(input, pos = 0) {
|
|
76
|
+
if (pos + TAR_BLOCK > input.length)
|
|
77
|
+
throw new TarReaderError("truncated", `tar truncated at offset ${pos}`);
|
|
78
|
+
if (isZeroBlock(input, pos))
|
|
79
|
+
return null;
|
|
80
|
+
if (!verifyChecksum(input, pos))
|
|
81
|
+
throw new TarReaderError("bad_checksum", `tar header checksum mismatch at offset ${pos}`);
|
|
82
|
+
const magic = readCstr(input, pos + 257, 6);
|
|
83
|
+
if (magic !== "ustar")
|
|
84
|
+
throw new TarReaderError("unsupported_format", `unexpected tar magic ${JSON.stringify(magic)} at offset ${pos}`);
|
|
85
|
+
const typeflag = input[pos + 156];
|
|
86
|
+
if (typeflag !== 0 && typeflag !== 0x30)
|
|
87
|
+
throw new TarReaderError("unsupported_format", `unsupported tar typeflag 0x${typeflag.toString(16)} at offset ${pos}`);
|
|
88
|
+
const name = readCstr(input, pos, 100);
|
|
89
|
+
const prefix = readCstr(input, pos + 345, 155);
|
|
90
|
+
const archivePath = prefix ? `${prefix}/${name}` : name;
|
|
91
|
+
if (isUnsafePath(archivePath))
|
|
92
|
+
throw new TarReaderError("unsafe_path", `unsafe tar entry path: ${JSON.stringify(archivePath)}`);
|
|
93
|
+
return { archivePath, size: readOctal(input, pos + 124, 12) };
|
|
94
|
+
}
|
|
74
95
|
export function readTar(input) {
|
|
75
96
|
const entries = [];
|
|
76
97
|
let pos = 0;
|
|
77
98
|
while (pos < input.length) {
|
|
78
|
-
|
|
79
|
-
|
|
80
|
-
}
|
|
81
|
-
if (isZeroBlock(input, pos)) {
|
|
99
|
+
const header = readTarHeader(input, pos);
|
|
100
|
+
if (!header) {
|
|
82
101
|
// Standard end marker is two zero blocks. We accept a single zero block
|
|
83
102
|
// at the tail too — buildTar always writes two, but defensive parsers
|
|
84
103
|
// shouldn't crash on a slightly truncated trailer.
|
|
85
104
|
return entries;
|
|
86
105
|
}
|
|
87
|
-
|
|
88
|
-
throw new TarReaderError("bad_checksum", `tar header checksum mismatch at offset ${pos}`);
|
|
89
|
-
}
|
|
90
|
-
const magic = readCstr(input, pos + 257, 6);
|
|
91
|
-
if (magic !== "ustar") {
|
|
92
|
-
throw new TarReaderError("unsupported_format", `unexpected tar magic ${JSON.stringify(magic)} at offset ${pos}`);
|
|
93
|
-
}
|
|
94
|
-
const typeflag = input[pos + 156];
|
|
95
|
-
if (typeflag !== 0 && typeflag !== 0x30) {
|
|
96
|
-
throw new TarReaderError("unsupported_format", `unsupported tar typeflag 0x${typeflag.toString(16)} at offset ${pos}`);
|
|
97
|
-
}
|
|
98
|
-
const name = readCstr(input, pos, 100);
|
|
99
|
-
const prefix = readCstr(input, pos + 345, 155);
|
|
100
|
-
const archivePath = prefix ? `${prefix}/${name}` : name;
|
|
101
|
-
if (isUnsafePath(archivePath)) {
|
|
102
|
-
throw new TarReaderError("unsafe_path", `unsafe tar entry path: ${JSON.stringify(archivePath)}`);
|
|
103
|
-
}
|
|
104
|
-
const size = readOctal(input, pos + 124, 12);
|
|
106
|
+
const { archivePath, size } = header;
|
|
105
107
|
const dataStart = pos + TAR_BLOCK;
|
|
106
108
|
const dataEnd = dataStart + size;
|
|
107
109
|
if (dataEnd > input.length) {
|
|
@@ -0,0 +1,3 @@
|
|
|
1
|
+
/** Inspect only requested Markdown bodies and stream retained USTAR bytes.
|
|
2
|
+
* Asset data is never collected or copied into an archive-sized buffer. */
|
|
3
|
+
export declare function streamTar(input: ReadableStream<Uint8Array<ArrayBuffer>>, retain: (path: string) => boolean, inspect: (path: string) => boolean, onText: (path: string, text: string) => Promise<void>): ReadableStream<Uint8Array<ArrayBuffer>>;
|
|
@@ -0,0 +1,95 @@
|
|
|
1
|
+
import { readTarHeader, TarReaderError } from "./tar-reader.js";
|
|
2
|
+
const BLOCK = 512;
|
|
3
|
+
/** Inspect only requested Markdown bodies and stream retained USTAR bytes.
|
|
4
|
+
* Asset data is never collected or copied into an archive-sized buffer. */
|
|
5
|
+
export function streamTar(input, retain, inspect, onText) {
|
|
6
|
+
const reader = input.getReader();
|
|
7
|
+
async function* chunks() {
|
|
8
|
+
let pending = new Uint8Array(0);
|
|
9
|
+
let offset = 0;
|
|
10
|
+
async function next() {
|
|
11
|
+
while (offset === pending.length) {
|
|
12
|
+
const item = await reader.read();
|
|
13
|
+
if (item.done)
|
|
14
|
+
return false;
|
|
15
|
+
pending = item.value;
|
|
16
|
+
offset = 0;
|
|
17
|
+
}
|
|
18
|
+
return true;
|
|
19
|
+
}
|
|
20
|
+
async function* take(length) {
|
|
21
|
+
while (length > 0) {
|
|
22
|
+
if (!(await next()))
|
|
23
|
+
throw new TarReaderError("truncated", "tar entry data truncated");
|
|
24
|
+
const count = Math.min(length, pending.length - offset);
|
|
25
|
+
const part = pending.subarray(offset, offset + count);
|
|
26
|
+
offset += count;
|
|
27
|
+
length -= count;
|
|
28
|
+
yield part;
|
|
29
|
+
}
|
|
30
|
+
}
|
|
31
|
+
try {
|
|
32
|
+
while (await next()) {
|
|
33
|
+
const header = new Uint8Array(BLOCK);
|
|
34
|
+
let position = 0;
|
|
35
|
+
for await (const part of take(BLOCK)) {
|
|
36
|
+
header.set(part, position);
|
|
37
|
+
position += part.length;
|
|
38
|
+
}
|
|
39
|
+
const entry = readTarHeader(header);
|
|
40
|
+
if (!entry) {
|
|
41
|
+
// Consume the gzip trailer too: checksum/format errors after the tar
|
|
42
|
+
// terminator must preserve the existing fail-closed filtering behavior.
|
|
43
|
+
while (!(await reader.read()).done) {
|
|
44
|
+
/* discard trailing tar bytes */
|
|
45
|
+
}
|
|
46
|
+
break;
|
|
47
|
+
}
|
|
48
|
+
const keep = retain(entry.archivePath);
|
|
49
|
+
const decoder = inspect(entry.archivePath) ? new TextDecoder() : null;
|
|
50
|
+
let text = "";
|
|
51
|
+
if (keep)
|
|
52
|
+
yield header;
|
|
53
|
+
for await (const part of take(entry.size)) {
|
|
54
|
+
if (decoder)
|
|
55
|
+
text += decoder.decode(part, { stream: true });
|
|
56
|
+
if (keep)
|
|
57
|
+
yield part;
|
|
58
|
+
}
|
|
59
|
+
if (decoder)
|
|
60
|
+
await onText(entry.archivePath, text + decoder.decode());
|
|
61
|
+
const padding = (BLOCK - (entry.size % BLOCK)) % BLOCK;
|
|
62
|
+
for await (const part of take(padding))
|
|
63
|
+
if (keep)
|
|
64
|
+
yield part;
|
|
65
|
+
}
|
|
66
|
+
yield new Uint8Array(BLOCK * 2);
|
|
67
|
+
}
|
|
68
|
+
finally {
|
|
69
|
+
await reader.cancel().catch(() => { });
|
|
70
|
+
reader.releaseLock();
|
|
71
|
+
}
|
|
72
|
+
}
|
|
73
|
+
const iterator = chunks();
|
|
74
|
+
return new ReadableStream({
|
|
75
|
+
async pull(controller) {
|
|
76
|
+
try {
|
|
77
|
+
const result = await iterator.next();
|
|
78
|
+
if (result.done)
|
|
79
|
+
controller.close();
|
|
80
|
+
else
|
|
81
|
+
controller.enqueue(result.value);
|
|
82
|
+
}
|
|
83
|
+
catch (error) {
|
|
84
|
+
controller.error(error);
|
|
85
|
+
}
|
|
86
|
+
},
|
|
87
|
+
async cancel(reason) {
|
|
88
|
+
// Generator.return queues behind a pending next(). Cancel the upstream
|
|
89
|
+
// read first so a stalled stream can reach the generator's finally block.
|
|
90
|
+
await reader.cancel(reason).catch(() => { });
|
|
91
|
+
await iterator.return(undefined);
|
|
92
|
+
reader.releaseLock();
|
|
93
|
+
},
|
|
94
|
+
});
|
|
95
|
+
}
|
package/dist/import/tar.js
CHANGED
|
@@ -54,7 +54,7 @@ export function splitUstarPath(archivePath) {
|
|
|
54
54
|
function ustarHeader(archivePath, size) {
|
|
55
55
|
const split = splitUstarPath(archivePath);
|
|
56
56
|
if (split === null) {
|
|
57
|
-
throw new Error(`archive path too long for USTAR (max ${
|
|
57
|
+
throw new Error(`archive path too long for USTAR (name max ${NAME_FIELD_MAX} UTF-8 bytes; optional prefix max ${PREFIX_FIELD_MAX} bytes split at /; PAX extended headers are not emitted): ${archivePath}`);
|
|
58
58
|
}
|
|
59
59
|
const header = new Uint8Array(TAR_BLOCK);
|
|
60
60
|
writeAscii(header, 0, 100, split.name);
|
package/dist/types/index.d.ts
CHANGED
|
@@ -234,6 +234,8 @@ export interface ImportResponse {
|
|
|
234
234
|
applied_at: string;
|
|
235
235
|
committed: number[];
|
|
236
236
|
rejected: RejectedPackage[];
|
|
237
|
+
/** Client preflight warnings; a matching body may have different metadata/assets. */
|
|
238
|
+
warnings?: RejectedPackage[];
|
|
237
239
|
}
|
|
238
240
|
export interface TaskHandle {
|
|
239
241
|
task_id: string;
|