@opendikw/web-client 0.1.0-rc.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (57) hide show
  1. package/LICENSE +21 -0
  2. package/README.md +31 -0
  3. package/dist/agent/agentClient.d.ts +26 -0
  4. package/dist/agent/agentClient.js +112 -0
  5. package/dist/agent/index.d.ts +1 -0
  6. package/dist/agent/index.js +1 -0
  7. package/dist/connection/index.d.ts +1 -0
  8. package/dist/connection/index.js +1 -0
  9. package/dist/convert/index.d.ts +1 -0
  10. package/dist/convert/index.js +1 -0
  11. package/dist/convert/mineru-convert.d.ts +83 -0
  12. package/dist/convert/mineru-convert.js +565 -0
  13. package/dist/core/client.d.ts +94 -0
  14. package/dist/core/client.js +386 -0
  15. package/dist/core/index.d.ts +2 -0
  16. package/dist/core/index.js +2 -0
  17. package/dist/core/ndjson.d.ts +9 -0
  18. package/dist/core/ndjson.js +57 -0
  19. package/dist/document/format.d.ts +17 -0
  20. package/dist/document/format.js +100 -0
  21. package/dist/document/index.d.ts +5 -0
  22. package/dist/document/index.js +5 -0
  23. package/dist/document/lang.d.ts +5 -0
  24. package/dist/document/lang.js +15 -0
  25. package/dist/document/markdown-blocks.d.ts +8 -0
  26. package/dist/document/markdown-blocks.js +169 -0
  27. package/dist/document/markdown.d.ts +32 -0
  28. package/dist/document/markdown.js +199 -0
  29. package/dist/document/source-inline-refs.d.ts +35 -0
  30. package/dist/document/source-inline-refs.js +242 -0
  31. package/dist/import/frontmatter-merge.d.ts +5 -0
  32. package/dist/import/frontmatter-merge.js +89 -0
  33. package/dist/import/import-bundle.d.ts +90 -0
  34. package/dist/import/import-bundle.js +336 -0
  35. package/dist/import/index.d.ts +5 -0
  36. package/dist/import/index.js +5 -0
  37. package/dist/import/kebab-source-name.d.ts +5 -0
  38. package/dist/import/kebab-source-name.js +31 -0
  39. package/dist/import/md-asset-refs.d.ts +27 -0
  40. package/dist/import/md-asset-refs.js +97 -0
  41. package/dist/import/tar-reader.d.ts +9 -0
  42. package/dist/import/tar-reader.js +114 -0
  43. package/dist/import/tar.d.ts +8 -0
  44. package/dist/import/tar.js +104 -0
  45. package/dist/translate/index.d.ts +1 -0
  46. package/dist/translate/index.js +1 -0
  47. package/dist/translate/translate.d.ts +69 -0
  48. package/dist/translate/translate.js +437 -0
  49. package/dist/types/agent.d.ts +82 -0
  50. package/dist/types/agent.js +1 -0
  51. package/dist/types/auth.d.ts +15 -0
  52. package/dist/types/auth.js +1 -0
  53. package/dist/types/index.d.ts +388 -0
  54. package/dist/types/index.js +3 -0
  55. package/dist/types/trace.d.ts +27 -0
  56. package/dist/types/trace.js +7 -0
  57. package/package.json +60 -0
@@ -0,0 +1,89 @@
1
+ // Merge a flat provenance block into a markdown file's YAML frontmatter at
2
+ // import time. We only ever ADD ``original_filename`` (and optionally
3
+ // ``converter``); a key the author already wrote is never clobbered, and the
4
+ // values are flat strings (the Base reader renders nested objects as JSON).
5
+ // Hand-rolled because the repo carries no YAML library. See
6
+ // docs/adr/0004-source-import-normalization.md.
7
+ /** Always double-quote so spaces / CJK / quotes round-trip. Mirrors the
8
+ * server-side ``yamlSafe`` in server/web/http.ts. */
9
+ function yamlSafe(value) {
10
+ return `"${value.replace(/\\/g, "\\\\").replace(/"/g, '\\"')}"`;
11
+ }
12
+ /** Top-level keys (``^key:``) of a frontmatter block body. Indented (nested)
13
+ * lines and longer key names are deliberately excluded. The colon may be
14
+ * followed by anything: a key the author wrote without a space (``key:value``)
15
+ * still counts as present, so we never append a duplicate that would make the
16
+ * whole block unparseable. */
17
+ function topLevelKeys(inner) {
18
+ const keys = new Set();
19
+ for (const line of inner.split(/\r?\n/)) {
20
+ const m = line.match(/^([A-Za-z0-9_]+):/);
21
+ if (m)
22
+ keys.add(m[1]);
23
+ }
24
+ return keys;
25
+ }
26
+ /** Does a ``---`` … ``---`` block body look like YAML frontmatter rather than a
27
+ * CommonMark thematic break wrapping prose? True if it is empty/all-blank or its
28
+ * first non-blank line is a mapping key (has a ``key:``) or a YAML comment. A
29
+ * leading ``---`` followed by prose with no colon (a horizontal rule) is not
30
+ * frontmatter, so we won't inject a key into the document body. */
31
+ function looksLikeFrontmatter(inner) {
32
+ for (const line of inner.split(/\r?\n/)) {
33
+ if (line.trim().length === 0)
34
+ continue;
35
+ return /^[ \t]*#/.test(line) || /^[ \t]*[A-Za-z0-9_][^:\r\n]*:/.test(line);
36
+ }
37
+ return true;
38
+ }
39
+ function wantedLines(fields) {
40
+ const lines = [["original_filename", fields.original_filename]];
41
+ if (fields.converter !== undefined)
42
+ lines.push(["converter", fields.converter]);
43
+ return lines;
44
+ }
45
+ /** Matches a terminated frontmatter block: opening ``---`` + EOL, lazy body,
46
+ * EOL + closing ``---``, optional trailing EOL. */
47
+ const FRONTMATTER = /^---(\r?\n)([\s\S]*?)(\r?\n)---(\r?\n?)/;
48
+ export function mergeFrontmatter(text, fields) {
49
+ const wanted = wantedLines(fields);
50
+ // Prepend a fresh frontmatter block in the file's dominant EOL.
51
+ const prepend = () => {
52
+ const eol = text.includes("\r\n") ? "\r\n" : "\n";
53
+ const body = wanted.map(([k, v]) => `${k}: ${yamlSafe(v)}`).join(eol);
54
+ return `---${eol}${body}${eol}---${eol}${text}`;
55
+ };
56
+ if (!text.startsWith("---"))
57
+ return prepend();
58
+ // An empty block (``---`` immediately followed by the closing ``---``) is a
59
+ // valid empty mapping but the FRONTMATTER regex can't match it (it needs a
60
+ // body line before the closing fence). Insert our keys between the fences.
61
+ const empty = text.match(/^---(\r?\n)---(\r?\n?)/);
62
+ if (empty) {
63
+ const eol = empty[1];
64
+ const body = wanted.map(([k, v]) => `${k}: ${yamlSafe(v)}`).join(eol);
65
+ return `---${eol}${body}${eol}---${empty[2]}${text.slice(empty[0].length)}`;
66
+ }
67
+ const m = text.match(FRONTMATTER);
68
+ if (!m) {
69
+ // Unterminated block — leave byte-identical (mirrors stripFrontmatter).
70
+ return text;
71
+ }
72
+ if (!looksLikeFrontmatter(m[2])) {
73
+ // The leading ``---`` is a CommonMark thematic break, not frontmatter —
74
+ // prepend a real block so we never inject a key into the document body.
75
+ return prepend();
76
+ }
77
+ const openEol = m[1];
78
+ const inner = m[2];
79
+ const closeEol = m[3];
80
+ const trailing = m[4];
81
+ const rest = text.slice(m[0].length);
82
+ const present = topLevelKeys(inner);
83
+ const missing = wanted.filter(([k]) => !present.has(k));
84
+ if (missing.length === 0)
85
+ return text;
86
+ const addition = missing.map(([k, v]) => `${k}: ${yamlSafe(v)}`).join(openEol);
87
+ const newInner = inner.length === 0 ? addition : `${inner}${openEol}${addition}`;
88
+ return `---${openEol}${newInner}${closeEol}---${trailing}${rest}`;
89
+ }
@@ -0,0 +1,90 @@
1
+ import { buildTar, splitUstarPath } from "./tar.js";
2
+ export { buildTar, splitUstarPath };
3
+ export declare const MD_EXTENSIONS: ReadonlySet<string>;
4
+ export declare const ASSET_EXTENSIONS: ReadonlySet<string>;
5
+ export interface ManifestFileEntry {
6
+ path: string;
7
+ size: number;
8
+ sha256: string;
9
+ }
10
+ export interface ManifestPackageEntry {
11
+ id: number;
12
+ md_path: string;
13
+ asset_paths: string[];
14
+ package_sha256: string;
15
+ }
16
+ export interface ManifestJson {
17
+ files: ManifestFileEntry[];
18
+ packages: ManifestPackageEntry[];
19
+ total_bytes: number;
20
+ }
21
+ export interface SkippedFile {
22
+ path: string;
23
+ reason: "unsupported_extension" | "empty_body" | "asset_missing" | "unreferenced_asset" | "duplicate_path" | "path_too_long";
24
+ detail?: string;
25
+ }
26
+ export interface ImportBundleResult {
27
+ payload: Blob;
28
+ manifestJson: string;
29
+ manifest: ManifestJson;
30
+ filesCount: number;
31
+ totalBytes: number;
32
+ skipped: SkippedFile[];
33
+ }
34
+ export interface BuildBundleOptions {
35
+ /** Maximum allowed total uncompressed bytes. Defaults to 256 MiB — the
36
+ * browser-realistic ceiling. Core itself accepts up to 1 GiB
37
+ * (``_DEFAULT_MAX_IMPORT_BYTES``) but we read every file fully into RAM
38
+ * twice (raw bytes + gzipped Blob) before POSTing, so anything close to
39
+ * that limit OOMs the tab. Streaming/spooling is a follow-up. */
40
+ maxTotalBytes?: number;
41
+ }
42
+ export declare function lowerExt(name: string): string;
43
+ /** Strip the common top-level directory segment from every file's
44
+ * ``webkitRelativePath``. When users pick a folder via ``webkitdirectory``
45
+ * the picked folder itself is included as the first segment, and we treat
46
+ * the *contents* as the project root.
47
+ *
48
+ * For files picked through ``<input multiple>`` (no relative path), the
49
+ * bare filename is used. */
50
+ export declare function computeProjectRelPath(file: File): string;
51
+ export declare function archivePath(projectRel: string): string;
52
+ export interface ScanResult {
53
+ /** Map from project-relative path → File, for every file we consider
54
+ * (md + asset; unsupported extensions are skipped). */
55
+ byProjectRel: Map<string, File>;
56
+ /** Project-relative paths of all .md files, in insertion order. */
57
+ mdPaths: string[];
58
+ skipped: SkippedFile[];
59
+ }
60
+ export declare function scanFiles(files: File[]): ScanResult;
61
+ /** Normalize a scanned import: rename each markdown to a Unicode-kebab basename,
62
+ * number distinct names that collapse to the same kebab, and merge a flat
63
+ * ``original_filename`` into its frontmatter (a no-op when it is already there,
64
+ * e.g. a MinerU-converted file). Assets are carried through unchanged — a plain
65
+ * ``.md`` is renamed by basename only (its dir, and therefore its relative asset
66
+ * refs, are untouched), and a MinerU unit's kebab path is idempotent. Runs after
67
+ * ``scanFiles`` so same-raw-path duplicates are already resolved. See
68
+ * docs/adr/0004-source-import-normalization.md. */
69
+ export declare function normalizeForImport(scan: ScanResult): Promise<ScanResult>;
70
+ export declare function sha256Hex(bytes: ArrayBuffer | Uint8Array): Promise<string>;
71
+ export declare function sha256HexString(text: string): Promise<string>;
72
+ /** ``sha256( sorted([mdSha, *assetShas]).join("\n").encode("ascii") )``
73
+ * — must agree byte-for-byte with ``md_inspect.package_sha256``. */
74
+ export declare function computePackageSha256(mdSha: string, assetShas: ReadonlyArray<string>): Promise<string>;
75
+ /** Inspect every md: extract refs, resolve them against the file inventory.
76
+ * Reports missing assets + empty-body cases via ``skipped`` and excludes
77
+ * those packages from the build. */
78
+ export declare function inspectMarkdownFiles(scan: ScanResult): Promise<{
79
+ packages: Array<{
80
+ mdProjectRel: string;
81
+ assetsProjectRel: string[];
82
+ }>;
83
+ skipped: SkippedFile[];
84
+ }>;
85
+ export declare function gzip(bytes: Uint8Array): Promise<Blob>;
86
+ export declare function buildImportBundle(files: File[], opts?: BuildBundleOptions): Promise<ImportBundleResult>;
87
+ export declare class ImportBundleError extends Error {
88
+ readonly code: string;
89
+ constructor(code: string, message: string);
90
+ }
@@ -0,0 +1,336 @@
1
+ // Browser-side equivalent of dikw-core/src/dikw_core/client/importer.py.
2
+ // Mirrors archive-path rules, manifest wire shape, and the package_sha256
3
+ // formula. Divergence shows up as ``manifest_package_sha256_mismatch`` from
4
+ // the server, so this file must stay aligned with the Python source of truth.
5
+ import { mergeFrontmatter } from "./frontmatter-merge.js";
6
+ import { kebabSourceName, kebabStem } from "./kebab-source-name.js";
7
+ import { extractAssetRefs, isRemoteRef, resolveAssetRef, stripFrontmatter, } from "./md-asset-refs.js";
8
+ import { buildTar, splitUstarPath } from "./tar.js";
9
+ export { buildTar, splitUstarPath };
10
+ export const MD_EXTENSIONS = new Set([".md"]);
11
+ export const ASSET_EXTENSIONS = new Set([
12
+ ".png",
13
+ ".jpg",
14
+ ".jpeg",
15
+ ".webp",
16
+ ".gif",
17
+ ".svg",
18
+ ".pdf",
19
+ ]);
20
+ const DEFAULT_MAX_TOTAL_BYTES = 256 * 1024 * 1024;
21
+ export function lowerExt(name) {
22
+ const i = name.lastIndexOf(".");
23
+ return i < 0 ? "" : name.slice(i).toLowerCase();
24
+ }
25
+ /** Strip the common top-level directory segment from every file's
26
+ * ``webkitRelativePath``. When users pick a folder via ``webkitdirectory``
27
+ * the picked folder itself is included as the first segment, and we treat
28
+ * the *contents* as the project root.
29
+ *
30
+ * For files picked through ``<input multiple>`` (no relative path), the
31
+ * bare filename is used. */
32
+ export function computeProjectRelPath(file) {
33
+ const raw = file.webkitRelativePath || file.name;
34
+ // Normalize separators just in case (some browsers / shells use ``\``).
35
+ const norm = raw.replace(/\\+/g, "/");
36
+ const slash = norm.indexOf("/");
37
+ if (slash < 0)
38
+ return norm;
39
+ return norm.slice(slash + 1);
40
+ }
41
+ export function archivePath(projectRel) {
42
+ if (projectRel === "sources" || projectRel.startsWith("sources/")) {
43
+ return projectRel;
44
+ }
45
+ return "sources/" + projectRel;
46
+ }
47
+ export function scanFiles(files) {
48
+ const byProjectRel = new Map();
49
+ const mdPaths = [];
50
+ const skipped = [];
51
+ // Detect collisions where two distinct ``File`` inputs strip to the same
52
+ // project-rel path. Without this, mdPaths gets duplicate entries and the
53
+ // server rejects the whole import with ``manifest_duplicate_md_path`` —
54
+ // the user just sees 'Import failed' with no actionable detail.
55
+ for (const file of files) {
56
+ const rel = computeProjectRelPath(file);
57
+ const ext = lowerExt(rel);
58
+ if (byProjectRel.has(rel)) {
59
+ const existing = byProjectRel.get(rel);
60
+ // Only flag if it's actually a different File object — picking the
61
+ // same File twice via two pickers should be a quiet no-op.
62
+ if (existing !== file) {
63
+ skipped.push({
64
+ path: rel,
65
+ reason: "duplicate_path",
66
+ detail: `${existing.size}B vs ${file.size}B`,
67
+ });
68
+ }
69
+ continue;
70
+ }
71
+ if (MD_EXTENSIONS.has(ext)) {
72
+ byProjectRel.set(rel, file);
73
+ mdPaths.push(rel);
74
+ }
75
+ else if (ASSET_EXTENSIONS.has(ext)) {
76
+ byProjectRel.set(rel, file);
77
+ }
78
+ else {
79
+ skipped.push({ path: rel, reason: "unsupported_extension" });
80
+ }
81
+ }
82
+ return { byProjectRel, mdPaths, skipped };
83
+ }
84
+ /** Append a ``-N`` disambiguation suffix to a kebab stem, capping the result at
85
+ * 28 code points so ``<stem>.md`` stays under 32. */
86
+ function appendSuffix(stem, n) {
87
+ const suffix = `-${n}`;
88
+ const max = 28 - suffix.length;
89
+ const capped = Array.from(stem).slice(0, max).join("").replace(/-+$/g, "");
90
+ return `${capped}${suffix}`;
91
+ }
92
+ /** Normalize a scanned import: rename each markdown to a Unicode-kebab basename,
93
+ * number distinct names that collapse to the same kebab, and merge a flat
94
+ * ``original_filename`` into its frontmatter (a no-op when it is already there,
95
+ * e.g. a MinerU-converted file). Assets are carried through unchanged — a plain
96
+ * ``.md`` is renamed by basename only (its dir, and therefore its relative asset
97
+ * refs, are untouched), and a MinerU unit's kebab path is idempotent. Runs after
98
+ * ``scanFiles`` so same-raw-path duplicates are already resolved. See
99
+ * docs/adr/0004-source-import-normalization.md. */
100
+ export async function normalizeForImport(scan) {
101
+ const byProjectRel = new Map();
102
+ const mdPaths = [];
103
+ const taken = new Set();
104
+ const oldMd = new Set(scan.mdPaths);
105
+ for (const mdPath of scan.mdPaths) {
106
+ const slash = mdPath.lastIndexOf("/");
107
+ const dir = slash < 0 ? "" : mdPath.slice(0, slash + 1); // keeps the trailing '/'
108
+ const base = slash < 0 ? mdPath : mdPath.slice(slash + 1);
109
+ const file = scan.byProjectRel.get(mdPath);
110
+ let newPath = `${dir}${kebabSourceName(base)}`;
111
+ for (let n = 2; taken.has(newPath); n++) {
112
+ newPath = `${dir}${appendSuffix(kebabStem(base), n)}.md`;
113
+ }
114
+ taken.add(newPath);
115
+ const merged = mergeFrontmatter(await file.text(), { original_filename: base });
116
+ const newName = newPath.slice(newPath.lastIndexOf("/") + 1);
117
+ byProjectRel.set(newPath, new File([merged], newName, { type: "text/markdown" }));
118
+ mdPaths.push(newPath);
119
+ }
120
+ // Carry every non-markdown file (assets) through verbatim.
121
+ for (const [path, file] of scan.byProjectRel) {
122
+ if (oldMd.has(path))
123
+ continue;
124
+ byProjectRel.set(path, file);
125
+ }
126
+ return { byProjectRel, mdPaths, skipped: [...scan.skipped] };
127
+ }
128
+ export async function sha256Hex(bytes) {
129
+ // ``crypto.subtle.digest`` accepts BufferSource (ArrayBuffer | ArrayBufferView)
130
+ // but the new ``Uint8Array<ArrayBufferLike>`` typing in TS 5.7+ doesn't satisfy
131
+ // the older ``BufferSource`` union (SharedArrayBuffer variance). Build a
132
+ // copy on a fresh ArrayBuffer to keep the type precise.
133
+ const len = bytes instanceof Uint8Array ? bytes.byteLength : bytes.byteLength;
134
+ const buf = new ArrayBuffer(len);
135
+ const view = new Uint8Array(buf);
136
+ if (bytes instanceof Uint8Array) {
137
+ view.set(bytes);
138
+ }
139
+ else {
140
+ view.set(new Uint8Array(bytes));
141
+ }
142
+ const digest = await crypto.subtle.digest("SHA-256", buf);
143
+ return hex(new Uint8Array(digest));
144
+ }
145
+ export async function sha256HexString(text) {
146
+ const data = new TextEncoder().encode(text);
147
+ const digest = await crypto.subtle.digest("SHA-256", data);
148
+ return hex(new Uint8Array(digest));
149
+ }
150
+ function hex(bytes) {
151
+ let out = "";
152
+ for (let i = 0; i < bytes.length; i++) {
153
+ out += bytes[i].toString(16).padStart(2, "0");
154
+ }
155
+ return out;
156
+ }
157
+ /** ``sha256( sorted([mdSha, *assetShas]).join("\n").encode("ascii") )``
158
+ * — must agree byte-for-byte with ``md_inspect.package_sha256``. */
159
+ export function computePackageSha256(mdSha, assetShas) {
160
+ const sorted = [mdSha, ...assetShas].slice().sort();
161
+ return sha256HexString(sorted.join("\n"));
162
+ }
163
+ /** Inspect every md: extract refs, resolve them against the file inventory.
164
+ * Reports missing assets + empty-body cases via ``skipped`` and excludes
165
+ * those packages from the build. */
166
+ export async function inspectMarkdownFiles(scan) {
167
+ const skipped = [...scan.skipped];
168
+ const packages = [];
169
+ const available = new Set(scan.byProjectRel.keys());
170
+ for (const mdRel of scan.mdPaths) {
171
+ const file = scan.byProjectRel.get(mdRel);
172
+ const text = await file.text();
173
+ const body = stripFrontmatter(text);
174
+ if (body.trim().length === 0) {
175
+ skipped.push({ path: mdRel, reason: "empty_body" });
176
+ continue;
177
+ }
178
+ const refs = extractAssetRefs(body);
179
+ const seen = new Set();
180
+ const assetsProjectRel = [];
181
+ let firstMissing = null;
182
+ for (const ref of refs) {
183
+ const resolved = resolveAssetRef(ref.originalPath, {
184
+ mdRelPath: mdRel,
185
+ available,
186
+ });
187
+ if (resolved === null) {
188
+ // ``http(s)://`` / ``data:`` / other non-file schemes are silently
189
+ // dropped — they're not local files. Anything else (including
190
+ // ``file:`` URIs, which core's ``_is_remote`` treats as local) is a
191
+ // genuine miss and gets reported.
192
+ if (!isRemoteRef(ref.originalPath)) {
193
+ if (firstMissing === null)
194
+ firstMissing = ref.originalPath;
195
+ }
196
+ continue;
197
+ }
198
+ if (seen.has(resolved))
199
+ continue;
200
+ seen.add(resolved);
201
+ assetsProjectRel.push(resolved);
202
+ }
203
+ if (firstMissing !== null) {
204
+ skipped.push({
205
+ path: mdRel,
206
+ reason: "asset_missing",
207
+ detail: firstMissing,
208
+ });
209
+ continue;
210
+ }
211
+ packages.push({ mdProjectRel: mdRel, assetsProjectRel });
212
+ }
213
+ return { packages, skipped };
214
+ }
215
+ export async function gzip(bytes) {
216
+ if (typeof CompressionStream === "undefined") {
217
+ throw new Error("CompressionStream is not available in this browser; please upgrade to Chrome 80+, Firefox 113+, or Safari 16.4+.");
218
+ }
219
+ // Don't go through Blob.stream() — jsdom's Blob doesn't implement it. The
220
+ // Response(BufferSource).body ReadableStream is the portable path. Wrap in
221
+ // a Blob to satisfy ``BodyInit`` in the TS 5.7+ DOM typings (Uint8Array's
222
+ // generic ArrayBufferLike doesn't satisfy the older BufferSource union).
223
+ // BlobPart requires an ArrayBuffer-backed view; copy to a fresh buffer.
224
+ const buf = new ArrayBuffer(bytes.byteLength);
225
+ new Uint8Array(buf).set(bytes);
226
+ const source = new Response(new Blob([buf])).body;
227
+ const compressed = source.pipeThrough(new CompressionStream("gzip"));
228
+ return new Response(compressed).blob();
229
+ }
230
+ // ---- Top-level builder ----------------------------------------------------
231
+ export async function buildImportBundle(files, opts = {}) {
232
+ const max = opts.maxTotalBytes ?? DEFAULT_MAX_TOTAL_BYTES;
233
+ const scan = await normalizeForImport(scanFiles(files));
234
+ const { packages, skipped } = await inspectMarkdownFiles(scan);
235
+ if (packages.length === 0) {
236
+ throw new ImportBundleError("no_packages", "No importable markdown files were found. Each .md must have a non-empty body and resolvable asset references.");
237
+ }
238
+ // Collect every unique archive path that participates in any package.
239
+ const projectRelByArchive = new Map(); // archive → project-rel
240
+ const referencedAssets = new Set();
241
+ for (const pkg of packages) {
242
+ projectRelByArchive.set(archivePath(pkg.mdProjectRel), pkg.mdProjectRel);
243
+ for (const a of pkg.assetsProjectRel) {
244
+ projectRelByArchive.set(archivePath(a), a);
245
+ referencedAssets.add(a);
246
+ }
247
+ }
248
+ // Warn (but don't block) on assets the user selected that no md references.
249
+ // Core's CLI importer rejects these outright; the web flow is more permissive
250
+ // (the user may have selected a broad folder and we shouldn't force them to
251
+ // unselect every loose image), so we surface them in ``skipped`` and leave
252
+ // them out of the bundle — see CLAUDE.md / plan for the divergence rationale.
253
+ for (const [projRel, file] of scan.byProjectRel) {
254
+ const ext = lowerExt(projRel);
255
+ if (!ASSET_EXTENSIONS.has(ext))
256
+ continue;
257
+ if (referencedAssets.has(projRel))
258
+ continue;
259
+ skipped.push({
260
+ path: projRel,
261
+ reason: "unreferenced_asset",
262
+ detail: `${file.size}B`,
263
+ });
264
+ }
265
+ // Hash + size each unique file once, in archive-path order so the manifest is
266
+ // stable (matches importer.py:_build_bundle sorted iteration).
267
+ const sortedArchive = Array.from(projectRelByArchive.keys()).sort();
268
+ // Pre-flight: sum ``File.size`` (no I/O — the browser already knows the
269
+ // sizes from the picker) and reject before loading anything into RAM. The
270
+ // old code called ``await file.arrayBuffer()`` first and only then checked
271
+ // ``totalBytes``, so a single oversized file could OOM the tab before the
272
+ // limit fired.
273
+ let totalBytes = 0;
274
+ for (const archive of sortedArchive) {
275
+ const projectRel = projectRelByArchive.get(archive);
276
+ const file = scan.byProjectRel.get(projectRel);
277
+ totalBytes += file.size;
278
+ }
279
+ if (totalBytes > max) {
280
+ throw new ImportBundleError("too_large", `Selected files total ${totalBytes} bytes, exceeding the ${max}-byte limit.`);
281
+ }
282
+ const entries = [];
283
+ for (const archive of sortedArchive) {
284
+ const projectRel = projectRelByArchive.get(archive);
285
+ const file = scan.byProjectRel.get(projectRel);
286
+ const buf = await file.arrayBuffer();
287
+ const data = new Uint8Array(buf);
288
+ const sha = await sha256Hex(buf);
289
+ entries.push({ archivePath: archive, data, sha, size: data.length });
290
+ }
291
+ const shaByArchive = new Map(entries.map((e) => [e.archivePath, e.sha]));
292
+ // Build manifest: files sorted by archive path; packages indexed by md order.
293
+ const manifestFiles = entries.map((e) => ({
294
+ path: e.archivePath,
295
+ size: e.size,
296
+ sha256: e.sha,
297
+ }));
298
+ const manifestPackages = [];
299
+ for (let i = 0; i < packages.length; i++) {
300
+ const pkg = packages[i];
301
+ const mdArchive = archivePath(pkg.mdProjectRel);
302
+ const assetArchives = pkg.assetsProjectRel.map(archivePath);
303
+ const mdSha = shaByArchive.get(mdArchive);
304
+ const assetShas = assetArchives.map((a) => shaByArchive.get(a));
305
+ const pkgSha = await computePackageSha256(mdSha, assetShas);
306
+ manifestPackages.push({
307
+ id: i,
308
+ md_path: mdArchive,
309
+ asset_paths: assetArchives,
310
+ package_sha256: pkgSha,
311
+ });
312
+ }
313
+ const manifest = {
314
+ files: manifestFiles,
315
+ packages: manifestPackages,
316
+ total_bytes: totalBytes,
317
+ };
318
+ const tarBytes = buildTar(entries.map((e) => ({ archivePath: e.archivePath, data: e.data })));
319
+ const payload = await gzip(tarBytes);
320
+ return {
321
+ payload,
322
+ manifestJson: JSON.stringify(manifest),
323
+ manifest,
324
+ filesCount: entries.length,
325
+ totalBytes,
326
+ skipped,
327
+ };
328
+ }
329
+ export class ImportBundleError extends Error {
330
+ code;
331
+ constructor(code, message) {
332
+ super(message);
333
+ this.name = "ImportBundleError";
334
+ this.code = code;
335
+ }
336
+ }
@@ -0,0 +1,5 @@
1
+ export * from "./import-bundle.js";
2
+ export * from "./frontmatter-merge.js";
3
+ export * from "./kebab-source-name.js";
4
+ export * from "./md-asset-refs.js";
5
+ export * from "./tar-reader.js";
@@ -0,0 +1,5 @@
1
+ export * from "./import-bundle.js";
2
+ export * from "./frontmatter-merge.js";
3
+ export * from "./kebab-source-name.js";
4
+ export * from "./md-asset-refs.js";
5
+ export * from "./tar-reader.js";
@@ -0,0 +1,5 @@
1
+ /** Kebab-case the stem (basename without its final extension), capped at
2
+ * ``maxStem`` code points. Never returns an empty string. */
3
+ export declare function kebabStem(name: string, maxStem?: number): string;
4
+ /** Kebab-case a source filename and force the ``.md`` extension. */
5
+ export declare function kebabSourceName(name: string): string;
@@ -0,0 +1,31 @@
1
+ // Normalize a source filename into a Unicode kebab-case stem at import time.
2
+ // ASCII is lowercased; runs of non-letter / non-number characters (spaces, ``_``,
3
+ // punctuation, hyphen variants like U+2010) collapse to a single ``-``; letters
4
+ // (including Han) and digits survive. The stem is capped by code point so the
5
+ // resulting ``<stem>.md`` filename stays well under 32 code points.
6
+ // See docs/adr/0004-source-import-normalization.md.
7
+ const MAX_STEM = 28;
8
+ /** Kebab-case the stem (basename without its final extension), capped at
9
+ * ``maxStem`` code points. Never returns an empty string. */
10
+ export function kebabStem(name, maxStem = MAX_STEM) {
11
+ const base = name.replace(/^.*[\\/]/, "");
12
+ const dot = base.lastIndexOf(".");
13
+ const stem = dot < 0 ? base : base.slice(0, dot);
14
+ const kebab = stem
15
+ .normalize("NFC")
16
+ .toLowerCase()
17
+ // A maximal run of non-letter/non-number chars becomes one hyphen, so no
18
+ // ``--`` can survive; letters (incl. Han), digits, and combining marks
19
+ // (``\p{M}`` — kept so a base+mark sequence with no precomposed form isn't
20
+ // fractured by a hyphen) are kept verbatim.
21
+ .replace(/[^\p{L}\p{N}\p{M}]+/gu, "-")
22
+ .replace(/^-+|-+$/g, "");
23
+ // Cap by code point (``Array.from`` never splits a surrogate pair), then
24
+ // re-trim any trailing hyphen the cut exposed.
25
+ const capped = Array.from(kebab).slice(0, maxStem).join("").replace(/-+$/g, "");
26
+ return capped || "untitled";
27
+ }
28
+ /** Kebab-case a source filename and force the ``.md`` extension. */
29
+ export function kebabSourceName(name) {
30
+ return `${kebabStem(name)}.md`;
31
+ }
@@ -0,0 +1,27 @@
1
+ export interface AssetRef {
2
+ originalPath: string;
3
+ alt: string;
4
+ start: number;
5
+ end: number;
6
+ syntax: "markdown" | "wikilink";
7
+ }
8
+ export declare function extractAssetRefs(body: string): AssetRef[];
9
+ export declare function isRemoteRef(originalPath: string): boolean;
10
+ /** Strip a YAML front-matter block (``---`` ... ``---``) off the body.
11
+ * Web side doesn't parse the YAML — we only need the body for asset extraction.
12
+ * If the front-matter block is unterminated we leave the text alone. */
13
+ export declare function stripFrontmatter(text: string): string;
14
+ /** POSIX-style join + normalize (handles ``..`` and ``.``, keeps no leading slash). */
15
+ export declare function posixJoinNormalize(base: string, rel: string): string;
16
+ export interface ResolveContext {
17
+ /** POSIX-style path of the source md file, relative to the project root
18
+ * (e.g. ``notes/foo.md``). The directory portion is the "sibling" base. */
19
+ mdRelPath: string;
20
+ /** Set of every selected file's POSIX-relative path under the project root. */
21
+ available: ReadonlySet<string>;
22
+ }
23
+ /** Sibling-of-md → project-root two-stage lookup, mirroring core's _resolve_local.
24
+ * Returns the POSIX-relative path under project root that the reference points at,
25
+ * or null if neither candidate exists in ``available``. Remote refs and refs that
26
+ * escape the project root return null and are treated as missing by the caller. */
27
+ export declare function resolveAssetRef(originalPath: string, ctx: ResolveContext): string | null;