@kernhq/module-quire 0.13.1 → 0.14.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/contract/models.d.ts +185 -0
- package/dist/contract/models.d.ts.map +1 -1
- package/dist/contract/models.js +133 -0
- package/dist/contract/models.js.map +1 -1
- package/dist/contract/permissions.d.ts +14 -0
- package/dist/contract/permissions.d.ts.map +1 -1
- package/dist/contract/permissions.js +82 -0
- package/dist/contract/permissions.js.map +1 -1
- package/dist/contract/properties.d.ts +4 -4
- package/dist/contract/router.d.ts +535 -8
- package/dist/contract/router.d.ts.map +1 -1
- package/dist/contract/router.js +133 -1
- package/dist/contract/router.js.map +1 -1
- package/dist/server/_impl.d.ts +519 -8
- package/dist/server/_impl.d.ts.map +1 -1
- package/dist/server/_impl.js +128 -0
- package/dist/server/_impl.js.map +1 -1
- package/dist/server/export/html.d.ts +67 -0
- package/dist/server/export/html.d.ts.map +1 -0
- package/dist/server/export/html.js +206 -0
- package/dist/server/export/html.js.map +1 -0
- package/dist/server/export/markdown.d.ts +51 -0
- package/dist/server/export/markdown.d.ts.map +1 -0
- package/dist/server/export/markdown.js +312 -0
- package/dist/server/export/markdown.js.map +1 -0
- package/dist/server/export/pdf.d.ts +20 -0
- package/dist/server/export/pdf.d.ts.map +1 -0
- package/dist/server/export/pdf.js +91 -0
- package/dist/server/export/pdf.js.map +1 -0
- package/dist/server/export/zip.d.ts +31 -0
- package/dist/server/export/zip.d.ts.map +1 -0
- package/dist/server/export/zip.js +158 -0
- package/dist/server/export/zip.js.map +1 -0
- package/dist/server/import/csv.d.ts +77 -0
- package/dist/server/import/csv.d.ts.map +1 -0
- package/dist/server/import/csv.js +263 -0
- package/dist/server/import/csv.js.map +1 -0
- package/dist/server/import/html.d.ts +52 -0
- package/dist/server/import/html.d.ts.map +1 -0
- package/dist/server/import/html.js +472 -0
- package/dist/server/import/html.js.map +1 -0
- package/dist/server/import/markdown.d.ts +63 -0
- package/dist/server/import/markdown.d.ts.map +1 -0
- package/dist/server/import/markdown.js +692 -0
- package/dist/server/import/markdown.js.map +1 -0
- package/dist/server/import/plan.d.ts +70 -0
- package/dist/server/import/plan.d.ts.map +1 -0
- package/dist/server/import/plan.js +761 -0
- package/dist/server/import/plan.js.map +1 -0
- package/dist/server/import/ydoc.d.ts +35 -0
- package/dist/server/import/ydoc.d.ts.map +1 -0
- package/dist/server/import/ydoc.js +91 -0
- package/dist/server/import/ydoc.js.map +1 -0
- package/dist/server/import/zip.d.ts +63 -0
- package/dist/server/import/zip.d.ts.map +1 -0
- package/dist/server/import/zip.js +308 -0
- package/dist/server/import/zip.js.map +1 -0
- package/dist/server/index.d.ts.map +1 -1
- package/dist/server/index.js +3 -1
- package/dist/server/index.js.map +1 -1
- package/dist/server/schema.d.ts +445 -1
- package/dist/server/schema.d.ts.map +1 -1
- package/dist/server/schema.js +146 -0
- package/dist/server/schema.js.map +1 -1
- package/dist/server/services/databases.d.ts +5 -5
- package/dist/server/services/export.d.ts +176 -0
- package/dist/server/services/export.d.ts.map +1 -0
- package/dist/server/services/export.js +822 -0
- package/dist/server/services/export.js.map +1 -0
- package/dist/server/services/import.d.ts +109 -0
- package/dist/server/services/import.d.ts.map +1 -0
- package/dist/server/services/import.js +570 -0
- package/dist/server/services/import.js.map +1 -0
- package/dist/server/services/index.d.ts +26 -1
- package/dist/server/services/index.d.ts.map +1 -1
- package/dist/server/services/index.js +60 -1
- package/dist/server/services/index.js.map +1 -1
- package/dist/server/services/versions.d.ts +1 -1
- package/migrations/0010_transfers.sql +154 -0
- package/migrations/meta/_journal.json +7 -0
- package/package.json +1 -1
- package/src/client/components/ExportDialog.svelte +685 -0
- package/src/client/components/ImportDialog.svelte +702 -0
- package/src/client/components/ImportReport.svelte +310 -0
- package/src/client/components/SidebarSpaces.svelte +79 -0
- package/src/client/i18n.ts +614 -0
- package/src/client/index.ts +32 -0
- package/src/client/mock.ts +318 -0
- package/src/client/module.ts +36 -0
- package/src/client/pages/PageView.svelte +36 -0
- package/src/client/pages/TransfersPage.svelte +570 -0
- package/src/client/permissions.ts +11 -0
- package/src/client/query.ts +23 -0
- package/src/client/transfers.ts +142 -0
- package/src/contract/models.ts +152 -0
- package/src/contract/permissions.ts +84 -0
- package/src/contract/router.ts +147 -0
|
@@ -0,0 +1,91 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* HTML to PDF, through Gotenberg.
|
|
3
|
+
*
|
|
4
|
+
* Gotenberg is a Chromium in a container, and it is already in the stack: `selfhost/docker-compose.yml`
|
|
5
|
+
* starts it under `--profile preview`, `dev/compose.yml` publishes it on 3500. Nothing in Kern had
|
|
6
|
+
* ever called it, so this is the first caller and the first thing that has to answer the question
|
|
7
|
+
* every optional dependency raises — **what happens when it is not there.**
|
|
8
|
+
*
|
|
9
|
+
* The answer is one sentence with three parts, and every one of them is load-bearing:
|
|
10
|
+
*
|
|
11
|
+
* - it never produces a half file. The bytes are read in full, in memory, before anything is
|
|
12
|
+
* written anywhere; a refusal, a timeout or a closed port throws before the caller has an
|
|
13
|
+
* artefact to store, so an export job goes to `failed` with `file_id` still null rather than to
|
|
14
|
+
* `done` with a truncated PDF a reader would find out about by opening it;
|
|
15
|
+
* - the failure names the URL it tried and the variable that changes it, because the person who
|
|
16
|
+
* reads it is an operator looking at a stack, not a developer looking at a stack trace. "PDF
|
|
17
|
+
* export could not reach Gotenberg at http://gotenberg:3000" is actionable; `ECONNREFUSED` is
|
|
18
|
+
* not;
|
|
19
|
+
* - it is bounded. Chromium rendering a five-hundred-page handbook can take minutes or can hang,
|
|
20
|
+
* and a job that hangs holds a worker for ever. The request carries a timeout and gives up.
|
|
21
|
+
*
|
|
22
|
+
* The default points at the compose service name rather than at nothing, so an instance that has the
|
|
23
|
+
* `preview` profile on needs no configuration at all and one that does not gets a refusal naming
|
|
24
|
+
* `GOTENBERG_URL`.
|
|
25
|
+
*/
|
|
26
|
+
import { KernError } from '@kernhq/kernel';
|
|
27
|
+
/** Where the compose stack puts it. `dev/compose.yml` publishes 3500 on the host. */
|
|
28
|
+
export const DEFAULT_GOTENBERG_URL = 'http://gotenberg:3000';
|
|
29
|
+
export const gotenbergUrl = () => process.env.GOTENBERG_URL?.trim() || DEFAULT_GOTENBERG_URL;
|
|
30
|
+
/** Long enough for a big handbook, short enough that a wedged Chromium is not a wedged worker. */
|
|
31
|
+
const DEFAULT_TIMEOUT_MS = 120_000;
|
|
32
|
+
/** A refusal's body is a diagnostic, not a document: enough to act on, never a page of HTML in a log. */
|
|
33
|
+
const REASON_LIMIT = 400;
|
|
34
|
+
/**
|
|
35
|
+
* A complete PDF, or a throw.
|
|
36
|
+
*
|
|
37
|
+
* The HTML has to be self-contained. Gotenberg's Chromium fetches whatever the document references,
|
|
38
|
+
* from inside its own container — so a presigned storage URL in an `<img src>` is both a network
|
|
39
|
+
* dependency this has no business having and a workspace uuid handed to a process that did not need
|
|
40
|
+
* it. Pictures reach here as `data:` URIs, put there by the caller.
|
|
41
|
+
*/
|
|
42
|
+
export async function htmlToPdf(html, options = {}) {
|
|
43
|
+
const base = (options.url ?? gotenbergUrl()).replace(/\/+$/, '');
|
|
44
|
+
const endpoint = `${base}/forms/chromium/convert/html`;
|
|
45
|
+
const form = new FormData();
|
|
46
|
+
// The part *must* be named `index.html`; Gotenberg renders that file and treats the rest as assets.
|
|
47
|
+
form.append('files', new Blob([html], { type: 'text/html' }), 'index.html');
|
|
48
|
+
form.append('paperWidth', '8.27');
|
|
49
|
+
form.append('paperHeight', '11.7');
|
|
50
|
+
form.append('marginTop', '0.6');
|
|
51
|
+
form.append('marginBottom', '0.6');
|
|
52
|
+
form.append('marginLeft', '0.6');
|
|
53
|
+
form.append('marginRight', '0.6');
|
|
54
|
+
form.append('printBackground', 'true');
|
|
55
|
+
// Without this a callout's tint and a code block's ground are dropped and the page loses its shape.
|
|
56
|
+
form.append('preferCssPageSize', 'false');
|
|
57
|
+
if (options.title)
|
|
58
|
+
form.append('metadata', JSON.stringify({ Title: options.title, Creator: 'Kern Quire' }));
|
|
59
|
+
let response;
|
|
60
|
+
try {
|
|
61
|
+
response = await fetch(endpoint, {
|
|
62
|
+
method: 'POST',
|
|
63
|
+
body: form,
|
|
64
|
+
signal: AbortSignal.timeout(options.timeoutMs ?? DEFAULT_TIMEOUT_MS),
|
|
65
|
+
});
|
|
66
|
+
}
|
|
67
|
+
catch (err) {
|
|
68
|
+
const why = err instanceof Error && err.name === 'TimeoutError' ? 'did not answer in time' : 'is not reachable';
|
|
69
|
+
throw new KernError('UNAVAILABLE', `PDF export needs Gotenberg, and ${base} ${why}. Start it (the self-host stack has it under ` +
|
|
70
|
+
'`--profile preview`) or point GOTENBERG_URL at one that is running.', { service: 'gotenberg', url: base });
|
|
71
|
+
}
|
|
72
|
+
if (!response.ok) {
|
|
73
|
+
const reason = await response.text().catch(() => '');
|
|
74
|
+
throw new KernError('UNAVAILABLE', `Gotenberg at ${base} refused to render this page (HTTP ${response.status}). ` +
|
|
75
|
+
`${reason.slice(0, REASON_LIMIT) || 'It gave no reason.'}`, { service: 'gotenberg', url: base, status: response.status });
|
|
76
|
+
}
|
|
77
|
+
const bytes = Buffer.from(await response.arrayBuffer());
|
|
78
|
+
/*
|
|
79
|
+
* A zero-length 200 is a real Gotenberg failure mode when Chromium dies mid-render, and it is the
|
|
80
|
+
* one that would otherwise be stored: an empty file, a job marked `done`, and a person who finds
|
|
81
|
+
* out by double-clicking it. `%PDF` is four bytes and settles it.
|
|
82
|
+
*/
|
|
83
|
+
if (bytes.length === 0 || bytes.subarray(0, 4).toString('latin1') !== '%PDF')
|
|
84
|
+
throw new KernError('UNAVAILABLE', `Gotenberg at ${base} returned something that is not a PDF`, {
|
|
85
|
+
service: 'gotenberg',
|
|
86
|
+
url: base,
|
|
87
|
+
bytes: bytes.length,
|
|
88
|
+
});
|
|
89
|
+
return bytes;
|
|
90
|
+
}
|
|
91
|
+
//# sourceMappingURL=pdf.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"pdf.js","sourceRoot":"","sources":["../../../src/server/export/pdf.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;;GAwBG;AACH,OAAO,EAAE,SAAS,EAAE,MAAM,gBAAgB,CAAA;AAE1C,qFAAqF;AACrF,MAAM,CAAC,MAAM,qBAAqB,GAAG,uBAAuB,CAAA;AAE5D,MAAM,CAAC,MAAM,YAAY,GAAG,GAAW,EAAE,CAAC,OAAO,CAAC,GAAG,CAAC,aAAa,EAAE,IAAI,EAAE,IAAI,qBAAqB,CAAA;AAEpG,kGAAkG;AAClG,MAAM,kBAAkB,GAAG,OAAO,CAAA;AAElC,yGAAyG;AACzG,MAAM,YAAY,GAAG,GAAG,CAAA;AAUxB;;;;;;;GAOG;AACH,MAAM,CAAC,KAAK,UAAU,SAAS,CAAC,IAAY,EAAE,UAAsB,EAAE;IACpE,MAAM,IAAI,GAAG,CAAC,OAAO,CAAC,GAAG,IAAI,YAAY,EAAE,CAAC,CAAC,OAAO,CAAC,MAAM,EAAE,EAAE,CAAC,CAAA;IAChE,MAAM,QAAQ,GAAG,GAAG,IAAI,8BAA8B,CAAA;IAEtD,MAAM,IAAI,GAAG,IAAI,QAAQ,EAAE,CAAA;IAC3B,oGAAoG;IACpG,IAAI,CAAC,MAAM,CAAC,OAAO,EAAE,IAAI,IAAI,CAAC,CAAC,IAAI,CAAC,EAAE,EAAE,IAAI,EAAE,WAAW,EAAE,CAAC,EAAE,YAAY,CAAC,CAAA;IAC3E,IAAI,CAAC,MAAM,CAAC,YAAY,EAAE,MAAM,CAAC,CAAA;IACjC,IAAI,CAAC,MAAM,CAAC,aAAa,EAAE,MAAM,CAAC,CAAA;IAClC,IAAI,CAAC,MAAM,CAAC,WAAW,EAAE,KAAK,CAAC,CAAA;IAC/B,IAAI,CAAC,MAAM,CAAC,cAAc,EAAE,KAAK,CAAC,CAAA;IAClC,IAAI,CAAC,MAAM,CAAC,YAAY,EAAE,KAAK,CAAC,CAAA;IAChC,IAAI,CAAC,MAAM,CAAC,aAAa,EAAE,KAAK,CAAC,CAAA;IACjC,IAAI,CAAC,MAAM,CAAC,iBAAiB,EAAE,MAAM,CAAC,CAAA;IACtC,oGAAoG;IACpG,IAAI,CAAC,MAAM,CAAC,mBAAmB,EAAE,OAAO,CAAC,CAAA;IACzC,IAAI,OAAO,CAAC,KAAK;QAAE,IAAI,CAAC,MAAM,CAAC,UAAU,EAAE,IAAI,CAAC,SAAS,CAAC,EAAE,KAAK,EAAE,OAAO,CAAC,KAAK,EAAE,OAAO,EAAE,YAAY,EAAE,CAAC,CAAC,CAAA;IAE3G,IAAI,QAAkB,CAAA;IACtB,IAAI,CAAC;QACH,QAAQ,GAAG,MAAM,KAAK,CAAC,QAAQ,EAAE;YAC/B,MAAM,EAAE,MAAM;YACd,IAAI,EAAE,IAAI;YACV,MAAM,EAAE,WAAW,CAAC,OAAO,CAAC,OAAO,CAAC,SAAS,IAAI,kBAAkB,CAAC;SACrE,CAAC,CAAA;IACJ,CAAC;IAAC,OAAO,GAAG,EAAE,CAAC;QACb,MAAM,GAAG,GACP,GAAG,YAAY,KAAK,IAAI,GAAG,CAAC,IAAI,KAAK,cAAc,CAAC,CAAC,CAAC,wBAAwB,CAAC,CAAC,CAAC,kBAAkB,CAAA;QACrG,MAAM,IAAI,SAAS,CACjB,aAAa,EACb,mCAAmC,IAAI,IAAI,GAAG,+CAA+C;YAC3F,qEAAqE,EACvE,EAAE,OAAO,EAAE,WAAW,EAAE,GAAG,EAAE,IAAI,EAAE,CACpC,CAAA;IACH,CAAC;IAED,IAAI,CAAC,QAAQ,CAAC,EAAE,EAAE,CAAC;QACjB,MAAM,MAAM,GAAG,MAAM,QAAQ,CAAC,IAAI,EAAE,CAAC,KAAK,CAAC,GAAG,EAAE,CAAC,EAAE,CAAC,CAAA;QACpD,MAAM,IAAI,SAAS,CACjB,aAAa,EACb,gBAAgB,IAAI,sCAAsC,QAAQ,CAAC,MAAM,KAAK;YAC5E,GAAG,MAAM,CAAC,KAAK,CAAC,CAAC,EAAE,YAAY,CAAC,IAAI,oBAAoB,EAAE,EAC5D,EAAE,OAAO,EAAE,WAAW,EAAE,GAAG,EAAE,IAAI,EAAE,MAAM,EAAE,QAAQ,CAAC,MAAM,EAAE,CAC7D,CAAA;IACH,CAAC;IAED,MAAM,KAAK,GAAG,MAAM,CAAC,IAAI,CAAC,MAAM,QAAQ,CAAC,WAAW,EAAE,CAAC,CAAA;IACvD;;;;OAIG;IACH,IAAI,KAAK,CAAC,MAAM,KAAK,CAAC,IAAI,KAAK,CAAC,QAAQ,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,QAAQ,CAAC,QAAQ,CAAC,KAAK,MAAM;QAC1E,MAAM,IAAI,SAAS,CAAC,aAAa,EAAE,gBAAgB,IAAI,uCAAuC,EAAE;YAC9F,OAAO,EAAE,WAAW;YACpB,GAAG,EAAE,IAAI;YACT,KAAK,EAAE,KAAK,CAAC,MAAM;SACpB,CAAC,CAAA;IACJ,OAAO,KAAK,CAAA;AACd,CAAC"}
|
|
@@ -0,0 +1,31 @@
|
|
|
1
|
+
export declare function crc32(data: Uint8Array): number;
|
|
2
|
+
export interface ZipEntry {
|
|
3
|
+
/** the path inside the archive, `/`-separated and never absolute or `..`-relative */
|
|
4
|
+
path: string;
|
|
5
|
+
data: Uint8Array;
|
|
6
|
+
}
|
|
7
|
+
/** What a caller may not exceed, so a refusal can name the number it broke. */
|
|
8
|
+
export declare const ZIP_LIMITS: {
|
|
9
|
+
maxTotalBytes: number;
|
|
10
|
+
maxEntryBytes: number;
|
|
11
|
+
maxEntries: number;
|
|
12
|
+
};
|
|
13
|
+
/**
|
|
14
|
+
* A path that cannot escape the folder somebody unzips into.
|
|
15
|
+
*
|
|
16
|
+
* A page title becomes a folder name, and a title is user input: `../../etc/whatever` is a real
|
|
17
|
+
* archive somebody can be handed, and a tool that follows it writes outside the extraction
|
|
18
|
+
* directory. Every segment is cleaned rather than rejected, because refusing an export because
|
|
19
|
+
* somebody named a page `C:\` would be absurd — but the cleaning is total, so nothing survives that
|
|
20
|
+
* a reader could resolve upwards.
|
|
21
|
+
*/
|
|
22
|
+
export declare function safeZipPath(path: string): string;
|
|
23
|
+
/**
|
|
24
|
+
* Every entry, in one buffer.
|
|
25
|
+
*
|
|
26
|
+
* Sizes are known before anything is written, so no entry carries a data descriptor and no reader
|
|
27
|
+
* has to seek backwards — which is what makes the output openable by the strictest tools as well as
|
|
28
|
+
* by the forgiving ones.
|
|
29
|
+
*/
|
|
30
|
+
export declare function writeZip(entries: ZipEntry[], now?: Date): Buffer;
|
|
31
|
+
//# sourceMappingURL=zip.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"zip.d.ts","sourceRoot":"","sources":["../../../src/server/export/zip.ts"],"names":[],"mappings":"AAqCA,wBAAgB,KAAK,CAAC,IAAI,EAAE,UAAU,GAAG,MAAM,CAI9C;AAkBD,MAAM,WAAW,QAAQ;IACvB,qFAAqF;IACrF,IAAI,EAAE,MAAM,CAAA;IACZ,IAAI,EAAE,UAAU,CAAA;CACjB;AAED,+EAA+E;AAC/E,eAAO,MAAM,UAAU;;;;CAA4E,CAAA;AAEnG;;;;;;;;GAQG;AACH,wBAAgB,WAAW,CAAC,IAAI,EAAE,MAAM,GAAG,MAAM,CAchD;AAED;;;;;;GAMG;AACH,wBAAgB,QAAQ,CAAC,OAAO,EAAE,QAAQ,EAAE,EAAE,GAAG,OAAa,GAAG,MAAM,CA4EtE"}
|
|
@@ -0,0 +1,158 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* A zip container, written by hand.
|
|
3
|
+
*
|
|
4
|
+
* An export of a subtree or a space is a folder per page with its pictures beside it, which is a zip
|
|
5
|
+
* and cannot reasonably be anything else. Writing the container here rather than taking a dependency
|
|
6
|
+
* is a deliberate trade and worth stating: the format needed is the 1989 one — deflate or store, no
|
|
7
|
+
* encryption, no ZIP64, sizes known before the entry is written — and that is about a hundred lines
|
|
8
|
+
* of well-specified header. `archiver` and `jszip` are streaming libraries whose job is everything
|
|
9
|
+
* this does not need, and every module repository commits its own lockfile, so a dependency here is
|
|
10
|
+
* a lockfile refresh outside the workspace in six places before it is a line of code.
|
|
11
|
+
*
|
|
12
|
+
* The limits are the honest part of that trade, and they are checked rather than assumed. No ZIP64
|
|
13
|
+
* means an archive is refused above 4 GiB, an entry above 4 GiB, or 65,535 entries — refused with a
|
|
14
|
+
* message, never truncated into a file that unzips to something incomplete. An export that would hit
|
|
15
|
+
* one of those is a job that fails and says so, which is the rule the whole slice is built on.
|
|
16
|
+
*
|
|
17
|
+
* Everything is deflated except what deflate makes bigger, which is most already-compressed pictures.
|
|
18
|
+
* The comparison is done rather than guessed from the extension: a PNG of flat colour still shrinks.
|
|
19
|
+
*/
|
|
20
|
+
import { deflateRawSync } from 'node:zlib';
|
|
21
|
+
/** 0xFFFFFFFF. Above this an offset or a size needs ZIP64, which this writer does not emit. */
|
|
22
|
+
const MAX_32 = 0xffff_ffff;
|
|
23
|
+
/** The count field in the end-of-central-directory record is 16 bits. */
|
|
24
|
+
const MAX_ENTRIES = 0xffff;
|
|
25
|
+
/** `crc32` over the whole entry, which the reader checks — a wrong one is a "corrupt archive". */
|
|
26
|
+
const CRC_TABLE = (() => {
|
|
27
|
+
const table = new Int32Array(256);
|
|
28
|
+
for (let n = 0; n < 256; n++) {
|
|
29
|
+
let c = n;
|
|
30
|
+
for (let k = 0; k < 8; k++)
|
|
31
|
+
c = c & 1 ? 0xedb8_8320 ^ (c >>> 1) : c >>> 1;
|
|
32
|
+
table[n] = c;
|
|
33
|
+
}
|
|
34
|
+
return table;
|
|
35
|
+
})();
|
|
36
|
+
export function crc32(data) {
|
|
37
|
+
let c = 0xffff_ffff;
|
|
38
|
+
for (let i = 0; i < data.length; i++)
|
|
39
|
+
c = CRC_TABLE[(c ^ data[i]) & 0xff] ^ (c >>> 8);
|
|
40
|
+
return (c ^ 0xffff_ffff) >>> 0;
|
|
41
|
+
}
|
|
42
|
+
/**
|
|
43
|
+
* The MS-DOS date and time a zip entry carries, from a real clock.
|
|
44
|
+
*
|
|
45
|
+
* Not a constant: an archive whose every file is stamped 1980 looks broken in a file manager, and
|
|
46
|
+
* some restore tools treat it as such. The resolution is two seconds, which is the format's, and the
|
|
47
|
+
* clock is UTC because the format has no zone and a local one would make the same export produce
|
|
48
|
+
* different bytes on two machines.
|
|
49
|
+
*/
|
|
50
|
+
function dosStamp(at) {
|
|
51
|
+
const year = Math.max(1980, at.getUTCFullYear());
|
|
52
|
+
return {
|
|
53
|
+
time: (at.getUTCHours() << 11) | (at.getUTCMinutes() << 5) | (at.getUTCSeconds() >> 1),
|
|
54
|
+
date: ((year - 1980) << 9) | ((at.getUTCMonth() + 1) << 5) | at.getUTCDate(),
|
|
55
|
+
};
|
|
56
|
+
}
|
|
57
|
+
/** What a caller may not exceed, so a refusal can name the number it broke. */
|
|
58
|
+
export const ZIP_LIMITS = { maxTotalBytes: MAX_32, maxEntryBytes: MAX_32, maxEntries: MAX_ENTRIES };
|
|
59
|
+
/**
|
|
60
|
+
* A path that cannot escape the folder somebody unzips into.
|
|
61
|
+
*
|
|
62
|
+
* A page title becomes a folder name, and a title is user input: `../../etc/whatever` is a real
|
|
63
|
+
* archive somebody can be handed, and a tool that follows it writes outside the extraction
|
|
64
|
+
* directory. Every segment is cleaned rather than rejected, because refusing an export because
|
|
65
|
+
* somebody named a page `C:\` would be absurd — but the cleaning is total, so nothing survives that
|
|
66
|
+
* a reader could resolve upwards.
|
|
67
|
+
*/
|
|
68
|
+
export function safeZipPath(path) {
|
|
69
|
+
const segments = path
|
|
70
|
+
.split('/')
|
|
71
|
+
.map((segment) => segment
|
|
72
|
+
.normalize('NFC')
|
|
73
|
+
// biome-ignore lint/suspicious/noControlCharactersInRegex: control characters are exactly what a filename must not carry
|
|
74
|
+
.replace(/[\u0000-\u001f\u007f\\:*?"<>|]+/g, '-')
|
|
75
|
+
.replace(/^\.+$/, '-')
|
|
76
|
+
.replace(/^\s+|[\s.]+$/g, '')
|
|
77
|
+
.slice(0, 100))
|
|
78
|
+
.filter((segment) => segment.length > 0);
|
|
79
|
+
return segments.join('/');
|
|
80
|
+
}
|
|
81
|
+
/**
|
|
82
|
+
* Every entry, in one buffer.
|
|
83
|
+
*
|
|
84
|
+
* Sizes are known before anything is written, so no entry carries a data descriptor and no reader
|
|
85
|
+
* has to seek backwards — which is what makes the output openable by the strictest tools as well as
|
|
86
|
+
* by the forgiving ones.
|
|
87
|
+
*/
|
|
88
|
+
export function writeZip(entries, now = new Date()) {
|
|
89
|
+
if (entries.length > MAX_ENTRIES)
|
|
90
|
+
throw new Error(`This export has ${entries.length} files, and a zip may hold ${MAX_ENTRIES}`);
|
|
91
|
+
const { time, date } = dosStamp(now);
|
|
92
|
+
const locals = [];
|
|
93
|
+
const centrals = [];
|
|
94
|
+
let offset = 0;
|
|
95
|
+
for (const entry of entries) {
|
|
96
|
+
const name = Buffer.from(entry.path, 'utf8');
|
|
97
|
+
const raw = Buffer.from(entry.data.buffer, entry.data.byteOffset, entry.data.byteLength);
|
|
98
|
+
if (raw.length > MAX_32)
|
|
99
|
+
throw new Error(`${entry.path} is larger than 4 GB, which a zip cannot hold`);
|
|
100
|
+
const deflated = raw.length > 0 ? deflateRawSync(raw, { level: 6 }) : Buffer.alloc(0);
|
|
101
|
+
// Stored rather than deflated when deflate did not help — which is the normal case for a JPEG.
|
|
102
|
+
const compress = deflated.length < raw.length;
|
|
103
|
+
const body = compress ? deflated : raw;
|
|
104
|
+
const crc = crc32(raw);
|
|
105
|
+
if (offset > MAX_32 - body.length - name.length - 30)
|
|
106
|
+
throw new Error('This export is larger than 4 GB, which is more than a zip can address');
|
|
107
|
+
const local = Buffer.alloc(30);
|
|
108
|
+
local.writeUInt32LE(0x0403_4b50, 0);
|
|
109
|
+
local.writeUInt16LE(20, 4);
|
|
110
|
+
// 0x0800: the name below is UTF-8. Without it a Persian page title is mojibake on extraction.
|
|
111
|
+
local.writeUInt16LE(0x0800, 6);
|
|
112
|
+
local.writeUInt16LE(compress ? 8 : 0, 8);
|
|
113
|
+
local.writeUInt16LE(time, 10);
|
|
114
|
+
local.writeUInt16LE(date, 12);
|
|
115
|
+
local.writeUInt32LE(crc, 14);
|
|
116
|
+
local.writeUInt32LE(body.length, 18);
|
|
117
|
+
local.writeUInt32LE(raw.length, 22);
|
|
118
|
+
local.writeUInt16LE(name.length, 26);
|
|
119
|
+
local.writeUInt16LE(0, 28);
|
|
120
|
+
locals.push(local, name, body);
|
|
121
|
+
const central = Buffer.alloc(46);
|
|
122
|
+
central.writeUInt32LE(0x0201_4b50, 0);
|
|
123
|
+
// Made by a Unix zip (3), spec 2.0 — so the permission bits below are read as Unix ones.
|
|
124
|
+
central.writeUInt16LE(0x0314, 4);
|
|
125
|
+
central.writeUInt16LE(20, 6);
|
|
126
|
+
central.writeUInt16LE(0x0800, 8);
|
|
127
|
+
central.writeUInt16LE(compress ? 8 : 0, 10);
|
|
128
|
+
central.writeUInt16LE(time, 12);
|
|
129
|
+
central.writeUInt16LE(date, 14);
|
|
130
|
+
central.writeUInt32LE(crc, 16);
|
|
131
|
+
central.writeUInt32LE(body.length, 20);
|
|
132
|
+
central.writeUInt32LE(raw.length, 24);
|
|
133
|
+
central.writeUInt16LE(name.length, 28);
|
|
134
|
+
central.writeUInt16LE(0, 30);
|
|
135
|
+
central.writeUInt16LE(0, 32);
|
|
136
|
+
central.writeUInt16LE(0, 34);
|
|
137
|
+
central.writeUInt16LE(0, 36);
|
|
138
|
+
// 0o100644 in the high half: a regular file somebody owns and everyone may read.
|
|
139
|
+
// Multiplied rather than shifted — `<<` is a 32-bit *signed* operator, so `0o100644 << 16` is
|
|
140
|
+
// negative and `writeUInt32LE` throws on it. The whole archive fails to build, at the last entry.
|
|
141
|
+
central.writeUInt32LE(0o100644 * 0x1_0000, 38);
|
|
142
|
+
central.writeUInt32LE(offset, 42);
|
|
143
|
+
centrals.push(central, name);
|
|
144
|
+
offset += local.length + name.length + body.length;
|
|
145
|
+
}
|
|
146
|
+
const directory = Buffer.concat(centrals);
|
|
147
|
+
const end = Buffer.alloc(22);
|
|
148
|
+
end.writeUInt32LE(0x0605_4b50, 0);
|
|
149
|
+
end.writeUInt16LE(0, 4);
|
|
150
|
+
end.writeUInt16LE(0, 6);
|
|
151
|
+
end.writeUInt16LE(entries.length, 8);
|
|
152
|
+
end.writeUInt16LE(entries.length, 10);
|
|
153
|
+
end.writeUInt32LE(directory.length, 12);
|
|
154
|
+
end.writeUInt32LE(offset, 16);
|
|
155
|
+
end.writeUInt16LE(0, 20);
|
|
156
|
+
return Buffer.concat([...locals, directory, end]);
|
|
157
|
+
}
|
|
158
|
+
//# sourceMappingURL=zip.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"zip.js","sourceRoot":"","sources":["../../../src/server/export/zip.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;GAkBG;AACH,OAAO,EAAE,cAAc,EAAE,MAAM,WAAW,CAAA;AAE1C,+FAA+F;AAC/F,MAAM,MAAM,GAAG,WAAW,CAAA;AAC1B,yEAAyE;AACzE,MAAM,WAAW,GAAG,MAAM,CAAA;AAE1B,kGAAkG;AAClG,MAAM,SAAS,GAAG,CAAC,GAAG,EAAE;IACtB,MAAM,KAAK,GAAG,IAAI,UAAU,CAAC,GAAG,CAAC,CAAA;IACjC,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,GAAG,EAAE,CAAC,EAAE,EAAE,CAAC;QAC7B,IAAI,CAAC,GAAG,CAAC,CAAA;QACT,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,CAAC,EAAE,CAAC,EAAE;YAAE,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,WAAW,GAAG,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC,CAAA;QACzE,KAAK,CAAC,CAAC,CAAC,GAAG,CAAC,CAAA;IACd,CAAC;IACD,OAAO,KAAK,CAAA;AACd,CAAC,CAAC,EAAE,CAAA;AAEJ,MAAM,UAAU,KAAK,CAAC,IAAgB;IACpC,IAAI,CAAC,GAAG,WAAW,CAAA;IACnB,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,IAAI,CAAC,MAAM,EAAE,CAAC,EAAE;QAAE,CAAC,GAAG,SAAS,CAAC,CAAC,CAAC,GAAG,IAAI,CAAC,CAAC,CAAE,CAAC,GAAG,IAAI,CAAE,GAAG,CAAC,CAAC,KAAK,CAAC,CAAC,CAAA;IACvF,OAAO,CAAC,CAAC,GAAG,WAAW,CAAC,KAAK,CAAC,CAAA;AAChC,CAAC;AAED;;;;;;;GAOG;AACH,SAAS,QAAQ,CAAC,EAAQ;IACxB,MAAM,IAAI,GAAG,IAAI,CAAC,GAAG,CAAC,IAAI,EAAE,EAAE,CAAC,cAAc,EAAE,CAAC,CAAA;IAChD,OAAO;QACL,IAAI,EAAE,CAAC,EAAE,CAAC,WAAW,EAAE,IAAI,EAAE,CAAC,GAAG,CAAC,EAAE,CAAC,aAAa,EAAE,IAAI,CAAC,CAAC,GAAG,CAAC,EAAE,CAAC,aAAa,EAAE,IAAI,CAAC,CAAC;QACtF,IAAI,EAAE,CAAC,CAAC,IAAI,GAAG,IAAI,CAAC,IAAI,CAAC,CAAC,GAAG,CAAC,CAAC,EAAE,CAAC,WAAW,EAAE,GAAG,CAAC,CAAC,IAAI,CAAC,CAAC,GAAG,EAAE,CAAC,UAAU,EAAE;KAC7E,CAAA;AACH,CAAC;AAQD,+EAA+E;AAC/E,MAAM,CAAC,MAAM,UAAU,GAAG,EAAE,aAAa,EAAE,MAAM,EAAE,aAAa,EAAE,MAAM,EAAE,UAAU,EAAE,WAAW,EAAE,CAAA;AAEnG;;;;;;;;GAQG;AACH,MAAM,UAAU,WAAW,CAAC,IAAY;IACtC,MAAM,QAAQ,GAAG,IAAI;SAClB,KAAK,CAAC,GAAG,CAAC;SACV,GAAG,CAAC,CAAC,OAAO,EAAE,EAAE,CACf,OAAO;SACJ,SAAS,CAAC,KAAK,CAAC;QACjB,yHAAyH;SACxH,OAAO,CAAC,kCAAkC,EAAE,GAAG,CAAC;SAChD,OAAO,CAAC,OAAO,EAAE,GAAG,CAAC;SACrB,OAAO,CAAC,eAAe,EAAE,EAAE,CAAC;SAC5B,KAAK,CAAC,CAAC,EAAE,GAAG,CAAC,CACjB;SACA,MAAM,CAAC,CAAC,OAAO,EAAE,EAAE,CAAC,OAAO,CAAC,MAAM,GAAG,CAAC,CAAC,CAAA;IAC1C,OAAO,QAAQ,CAAC,IAAI,CAAC,GAAG,CAAC,CAAA;AAC3B,CAAC;AAED;;;;;;GAMG;AACH,MAAM,UAAU,QAAQ,CAAC,OAAmB,EAAE,GAAG,GAAG,IAAI,IAAI,EAAE;IAC5D,IAAI,OAAO,CAAC,MAAM,GAAG,WAAW;QAC9B,MAAM,IAAI,KAAK,CAAC,mBAAmB,OAAO,CAAC,MAAM,8BAA8B,WAAW,EAAE,CAAC,CAAA;IAE/F,MAAM,EAAE,IAAI,EAAE,IAAI,EAAE,GAAG,QAAQ,CAAC,GAAG,CAAC,CAAA;IACpC,MAAM,MAAM,GAAa,EAAE,CAAA;IAC3B,MAAM,QAAQ,GAAa,EAAE,CAAA;IAC7B,IAAI,MAAM,GAAG,CAAC,CAAA;IAEd,KAAK,MAAM,KAAK,IAAI,OAAO,EAAE,CAAC;QAC5B,MAAM,IAAI,GAAG,MAAM,CAAC,IAAI,CAAC,KAAK,CAAC,IAAI,EAAE,MAAM,CAAC,CAAA;QAC5C,MAAM,GAAG,GAAG,MAAM,CAAC,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,MAAM,EAAE,KAAK,CAAC,IAAI,CAAC,UAAU,EAAE,KAAK,CAAC,IAAI,CAAC,UAAU,CAAC,CAAA;QACxF,IAAI,GAAG,CAAC,MAAM,GAAG,MAAM;YAAE,MAAM,IAAI,KAAK,CAAC,GAAG,KAAK,CAAC,IAAI,+CAA+C,CAAC,CAAA;QACtG,MAAM,QAAQ,GAAG,GAAG,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,CAAC,cAAc,CAAC,GAAG,EAAE,EAAE,KAAK,EAAE,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC,CAAC,CAAA;QACrF,+FAA+F;QAC/F,MAAM,QAAQ,GAAG,QAAQ,CAAC,MAAM,GAAG,GAAG,CAAC,MAAM,CAAA;QAC7C,MAAM,IAAI,GAAG,QAAQ,CAAC,CAAC,CAAC,QAAQ,CAAC,CAAC,CAAC,GAAG,CAAA;QACtC,MAAM,GAAG,GAAG,KAAK,CAAC,GAAG,CAAC,CAAA;QAEtB,IAAI,MAAM,GAAG,MAAM,GAAG,IAAI,CAAC,MAAM,GAAG,IAAI,CAAC,MAAM,GAAG,EAAE;YAClD,MAAM,IAAI,KAAK,CAAC,uEAAuE,CAAC,CAAA;QAE1F,MAAM,KAAK,GAAG,MAAM,CAAC,KAAK,CAAC,EAAE,CAAC,CAAA;QAC9B,KAAK,CAAC,aAAa,CAAC,WAAW,EAAE,CAAC,CAAC,CAAA;QACnC,KAAK,CAAC,aAAa,CAAC,EAAE,EAAE,CAAC,CAAC,CAAA;QAC1B,8FAA8F;QAC9F,KAAK,CAAC,aAAa,CAAC,MAAM,EAAE,CAAC,CAAC,CAAA;QAC9B,KAAK,CAAC,aAAa,CAAC,QAAQ,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAA;QACxC,KAAK,CAAC,aAAa,CAAC,IAAI,EAAE,EAAE,CAAC,CAAA;QAC7B,KAAK,CAAC,aAAa,CAAC,IAAI,EAAE,EAAE,CAAC,CAAA;QAC7B,KAAK,CAAC,aAAa,CAAC,GAAG,EAAE,EAAE,CAAC,CAAA;QAC5B,KAAK,CAAC,aAAa,CAAC,IAAI,CAAC,MAAM,EAAE,EAAE,CAAC,CAAA;QACpC,KAAK,CAAC,aAAa,CAAC,GAAG,CAAC,MAAM,EAAE,EAAE,CAAC,CAAA;QACnC,KAAK,CAAC,aAAa,CAAC,IAAI,CAAC,MAAM,EAAE,EAAE,CAAC,CAAA;QACpC,KAAK,CAAC,aAAa,CAAC,CAAC,EAAE,EAAE,CAAC,CAAA;QAC1B,MAAM,CAAC,IAAI,CAAC,KAAK,EAAE,IAAI,EAAE,IAAI,CAAC,CAAA;QAE9B,MAAM,OAAO,GAAG,MAAM,CAAC,KAAK,CAAC,EAAE,CAAC,CAAA;QAChC,OAAO,CAAC,aAAa,CAAC,WAAW,EAAE,CAAC,CAAC,CAAA;QACrC,yFAAyF;QACzF,OAAO,CAAC,aAAa,CAAC,MAAM,EAAE,CAAC,CAAC,CAAA;QAChC,OAAO,CAAC,aAAa,CAAC,EAAE,EAAE,CAAC,CAAC,CAAA;QAC5B,OAAO,CAAC,aAAa,CAAC,MAAM,EAAE,CAAC,CAAC,CAAA;QAChC,OAAO,CAAC,aAAa,CAAC,QAAQ,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAA;QAC3C,OAAO,CAAC,aAAa,CAAC,IAAI,EAAE,EAAE,CAAC,CAAA;QAC/B,OAAO,CAAC,aAAa,CAAC,IAAI,EAAE,EAAE,CAAC,CAAA;QAC/B,OAAO,CAAC,aAAa,CAAC,GAAG,EAAE,EAAE,CAAC,CAAA;QAC9B,OAAO,CAAC,aAAa,CAAC,IAAI,CAAC,MAAM,EAAE,EAAE,CAAC,CAAA;QACtC,OAAO,CAAC,aAAa,CAAC,GAAG,CAAC,MAAM,EAAE,EAAE,CAAC,CAAA;QACrC,OAAO,CAAC,aAAa,CAAC,IAAI,CAAC,MAAM,EAAE,EAAE,CAAC,CAAA;QACtC,OAAO,CAAC,aAAa,CAAC,CAAC,EAAE,EAAE,CAAC,CAAA;QAC5B,OAAO,CAAC,aAAa,CAAC,CAAC,EAAE,EAAE,CAAC,CAAA;QAC5B,OAAO,CAAC,aAAa,CAAC,CAAC,EAAE,EAAE,CAAC,CAAA;QAC5B,OAAO,CAAC,aAAa,CAAC,CAAC,EAAE,EAAE,CAAC,CAAA;QAC5B,iFAAiF;QACjF,8FAA8F;QAC9F,kGAAkG;QAClG,OAAO,CAAC,aAAa,CAAC,QAAQ,GAAG,QAAQ,EAAE,EAAE,CAAC,CAAA;QAC9C,OAAO,CAAC,aAAa,CAAC,MAAM,EAAE,EAAE,CAAC,CAAA;QACjC,QAAQ,CAAC,IAAI,CAAC,OAAO,EAAE,IAAI,CAAC,CAAA;QAE5B,MAAM,IAAI,KAAK,CAAC,MAAM,GAAG,IAAI,CAAC,MAAM,GAAG,IAAI,CAAC,MAAM,CAAA;IACpD,CAAC;IAED,MAAM,SAAS,GAAG,MAAM,CAAC,MAAM,CAAC,QAAQ,CAAC,CAAA;IACzC,MAAM,GAAG,GAAG,MAAM,CAAC,KAAK,CAAC,EAAE,CAAC,CAAA;IAC5B,GAAG,CAAC,aAAa,CAAC,WAAW,EAAE,CAAC,CAAC,CAAA;IACjC,GAAG,CAAC,aAAa,CAAC,CAAC,EAAE,CAAC,CAAC,CAAA;IACvB,GAAG,CAAC,aAAa,CAAC,CAAC,EAAE,CAAC,CAAC,CAAA;IACvB,GAAG,CAAC,aAAa,CAAC,OAAO,CAAC,MAAM,EAAE,CAAC,CAAC,CAAA;IACpC,GAAG,CAAC,aAAa,CAAC,OAAO,CAAC,MAAM,EAAE,EAAE,CAAC,CAAA;IACrC,GAAG,CAAC,aAAa,CAAC,SAAS,CAAC,MAAM,EAAE,EAAE,CAAC,CAAA;IACvC,GAAG,CAAC,aAAa,CAAC,MAAM,EAAE,EAAE,CAAC,CAAA;IAC7B,GAAG,CAAC,aAAa,CAAC,CAAC,EAAE,EAAE,CAAC,CAAA;IAExB,OAAO,MAAM,CAAC,MAAM,CAAC,CAAC,GAAG,MAAM,EAAE,SAAS,EAAE,GAAG,CAAC,CAAC,CAAA;AACnD,CAAC"}
|
|
@@ -0,0 +1,77 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* A CSV as a Quire database: columns with guessed types, and rows carrying values of those types.
|
|
3
|
+
*
|
|
4
|
+
* A Notion export writes one `.csv` per database — the same file you would get from a spreadsheet —
|
|
5
|
+
* and a CSV has no types at all. Importing every column as text is the safe answer and the useless
|
|
6
|
+
* one: a date column you cannot sort by date and a number column that sorts 10 before 9 are the two
|
|
7
|
+
* complaints a database import gets, and both are the same complaint.
|
|
8
|
+
*
|
|
9
|
+
* **So the type is guessed, and the guess is reported.** Every column contributes a sentence to the
|
|
10
|
+
* import report saying what it was read as and on what evidence — "Due: read as a date, from 12 of 12
|
|
11
|
+
* values" — because a guess nobody is told about is indistinguishable from a mistake. The rules are
|
|
12
|
+
* ordered from most specific to least and every one of them demands *every* non-empty value in the
|
|
13
|
+
* column agree; one stray `n/a` in a date column makes it text, which is the right way round. A
|
|
14
|
+
* column that is text is not a failure and does not say "failed": it is a column whose values did not
|
|
15
|
+
* all look like anything narrower.
|
|
16
|
+
*
|
|
17
|
+
* What this file does **not** do is guess a relation, a rollup, a formula or a person. Each of those
|
|
18
|
+
* names something outside the file — another database, another column, a member of this workspace —
|
|
19
|
+
* and inventing one from a string of text is how an import produces a database that looks right and
|
|
20
|
+
* computes nothing. They stay text, and the report says so.
|
|
21
|
+
*/
|
|
22
|
+
import type { PropertyConfig, PropertyType } from '../../contract/index.js';
|
|
23
|
+
/**
|
|
24
|
+
* RFC 4180, with the two departures every real file needs.
|
|
25
|
+
*
|
|
26
|
+
* A quoted field may hold newlines and `""` for a literal quote — that part is the specification. The
|
|
27
|
+
* departures are that a lone `\r` or a `\r\n` both end a record (Excel writes one, Notion the other)
|
|
28
|
+
* and that a stray quote inside an unquoted field is a character rather than an error, because the
|
|
29
|
+
* alternative is refusing a file every other reader opens.
|
|
30
|
+
*/
|
|
31
|
+
export declare function parseCsv(source: string): string[][];
|
|
32
|
+
/** `2026-01-05`, `January 5, 2026`, `05/01/2026` — as the ISO string a date cell stores. */
|
|
33
|
+
export declare function parseDateValue(raw: string): string | null;
|
|
34
|
+
/** `1,234.50`, `$99`, `12%` — as the number a numeric sort and a rollup can use. */
|
|
35
|
+
export declare function parseNumberValue(raw: string): {
|
|
36
|
+
value: number;
|
|
37
|
+
percent: boolean;
|
|
38
|
+
currency: string | null;
|
|
39
|
+
} | null;
|
|
40
|
+
export interface GuessedColumn {
|
|
41
|
+
name: string;
|
|
42
|
+
type: PropertyType;
|
|
43
|
+
config: PropertyConfig;
|
|
44
|
+
/**
|
|
45
|
+
* What the guess was and what it was made from, in one sentence for the report.
|
|
46
|
+
*
|
|
47
|
+
* Written here rather than at the call site because only this function knows the evidence — how
|
|
48
|
+
* many values agreed, how many distinct choices there were — and a reason assembled later would
|
|
49
|
+
* either repeat the rules or say less than it could.
|
|
50
|
+
*/
|
|
51
|
+
note: string;
|
|
52
|
+
}
|
|
53
|
+
/**
|
|
54
|
+
* One column's type, from its values.
|
|
55
|
+
*
|
|
56
|
+
* The order is the whole algorithm: checkbox before number (`1` and `0` are both), number before
|
|
57
|
+
* date (a bare year is a number), url and email before select (a column of six addresses is not six
|
|
58
|
+
* choices), multi-select before select (a comma is the only thing that separates them), and select
|
|
59
|
+
* before text (the fallback that is always true).
|
|
60
|
+
*/
|
|
61
|
+
export declare function guessColumn(name: string, rawValues: string[]): GuessedColumn;
|
|
62
|
+
/** `Design, Urgent` → `['Design', 'Urgent']`, which is how every exporter writes a multi-select. */
|
|
63
|
+
export declare const splitChoices: (value: string) => string[];
|
|
64
|
+
/**
|
|
65
|
+
* One cell, as the value the column's type stores.
|
|
66
|
+
*
|
|
67
|
+
* Null means "leave the key out of `props`" rather than "write null": an absent key is what every
|
|
68
|
+
* other writer in this module produces for an empty cell, and a null would make a `not_contains`
|
|
69
|
+
* filter behave differently for an imported row than for one somebody typed.
|
|
70
|
+
*
|
|
71
|
+
* `select` stores a **scalar** option id and `multi_select` an **array** of them, and the difference
|
|
72
|
+
* is load-bearing rather than cosmetic: `query.ts` compares a select as text through `props->>'key'`,
|
|
73
|
+
* so an array there stringifies to `["done"]` and matches no filter anybody can write. The cell
|
|
74
|
+
* component reads either shape, which is exactly why this is easy to get wrong and invisible on screen.
|
|
75
|
+
*/
|
|
76
|
+
export declare function coerceValue(column: GuessedColumn, raw: string): unknown;
|
|
77
|
+
//# sourceMappingURL=csv.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"csv.d.ts","sourceRoot":"","sources":["../../../src/server/import/csv.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;GAoBG;AACH,OAAO,KAAK,EAAE,cAAc,EAAE,YAAY,EAAgB,MAAM,yBAAyB,CAAA;AAWzF;;;;;;;GAOG;AACH,wBAAgB,QAAQ,CAAC,MAAM,EAAE,MAAM,GAAG,MAAM,EAAE,EAAE,CA4DnD;AAeD,4FAA4F;AAC5F,wBAAgB,cAAc,CAAC,GAAG,EAAE,MAAM,GAAG,MAAM,GAAG,IAAI,CAazD;AAED,oFAAoF;AACpF,wBAAgB,gBAAgB,CAC9B,GAAG,EAAE,MAAM,GACV;IAAE,KAAK,EAAE,MAAM,CAAC;IAAC,OAAO,EAAE,OAAO,CAAC;IAAC,QAAQ,EAAE,MAAM,GAAG,IAAI,CAAA;CAAE,GAAG,IAAI,CASrE;AAyBD,MAAM,WAAW,aAAa;IAC5B,IAAI,EAAE,MAAM,CAAA;IACZ,IAAI,EAAE,YAAY,CAAA;IAClB,MAAM,EAAE,cAAc,CAAA;IACtB;;;;;;OAMG;IACH,IAAI,EAAE,MAAM,CAAA;CACb;AAED;;;;;;;GAOG;AACH,wBAAgB,WAAW,CAAC,IAAI,EAAE,MAAM,EAAE,SAAS,EAAE,MAAM,EAAE,GAAG,aAAa,CAmF5E;AAED,oGAAoG;AACpG,eAAO,MAAM,YAAY,GAAI,OAAO,MAAM,KAAG,MAAM,EAIb,CAAA;AAEtC;;;;;;;;;;;GAWG;AACH,wBAAgB,WAAW,CAAC,MAAM,EAAE,aAAa,EAAE,GAAG,EAAE,MAAM,GAAG,OAAO,CAwBvE"}
|
|
@@ -0,0 +1,263 @@
|
|
|
1
|
+
/** Enough rows for the guess to mean something without reading a whole spreadsheet twice. */
|
|
2
|
+
const SAMPLE = 500;
|
|
3
|
+
/** Above this a column of distinct strings is prose, not a set of choices. */
|
|
4
|
+
const MAX_OPTIONS = 40;
|
|
5
|
+
/** A choice longer than this is a sentence somebody typed, not a label. */
|
|
6
|
+
const MAX_OPTION_LENGTH = 60;
|
|
7
|
+
/**
|
|
8
|
+
* RFC 4180, with the two departures every real file needs.
|
|
9
|
+
*
|
|
10
|
+
* A quoted field may hold newlines and `""` for a literal quote — that part is the specification. The
|
|
11
|
+
* departures are that a lone `\r` or a `\r\n` both end a record (Excel writes one, Notion the other)
|
|
12
|
+
* and that a stray quote inside an unquoted field is a character rather than an error, because the
|
|
13
|
+
* alternative is refusing a file every other reader opens.
|
|
14
|
+
*/
|
|
15
|
+
export function parseCsv(source) {
|
|
16
|
+
const rows = [];
|
|
17
|
+
let row = [];
|
|
18
|
+
let field = '';
|
|
19
|
+
let quoted = false;
|
|
20
|
+
let sawField = false;
|
|
21
|
+
const endField = () => {
|
|
22
|
+
row.push(field);
|
|
23
|
+
field = '';
|
|
24
|
+
sawField = false;
|
|
25
|
+
};
|
|
26
|
+
const endRow = () => {
|
|
27
|
+
endField();
|
|
28
|
+
rows.push(row);
|
|
29
|
+
row = [];
|
|
30
|
+
};
|
|
31
|
+
for (let i = 0; i < source.length; i++) {
|
|
32
|
+
const ch = source[i];
|
|
33
|
+
if (quoted) {
|
|
34
|
+
if (ch !== '"') {
|
|
35
|
+
field += ch;
|
|
36
|
+
continue;
|
|
37
|
+
}
|
|
38
|
+
if (source[i + 1] === '"') {
|
|
39
|
+
field += '"';
|
|
40
|
+
i++;
|
|
41
|
+
continue;
|
|
42
|
+
}
|
|
43
|
+
quoted = false;
|
|
44
|
+
continue;
|
|
45
|
+
}
|
|
46
|
+
if (ch === '"' && !sawField) {
|
|
47
|
+
quoted = true;
|
|
48
|
+
sawField = true;
|
|
49
|
+
continue;
|
|
50
|
+
}
|
|
51
|
+
if (ch === ',') {
|
|
52
|
+
endField();
|
|
53
|
+
continue;
|
|
54
|
+
}
|
|
55
|
+
if (ch === '\r') {
|
|
56
|
+
if (source[i + 1] === '\n')
|
|
57
|
+
i++;
|
|
58
|
+
endRow();
|
|
59
|
+
continue;
|
|
60
|
+
}
|
|
61
|
+
if (ch === '\n') {
|
|
62
|
+
endRow();
|
|
63
|
+
continue;
|
|
64
|
+
}
|
|
65
|
+
field += ch;
|
|
66
|
+
sawField = true;
|
|
67
|
+
}
|
|
68
|
+
// A file ending in a newline has already closed its last row; anything else is still open.
|
|
69
|
+
if (field.length > 0 || row.length > 0)
|
|
70
|
+
endRow();
|
|
71
|
+
// A trailing empty record is what a file ending in a newline leaves behind, and it is not a row.
|
|
72
|
+
while (rows.length > 0 && rows.at(-1).every((value) => value.trim() === ''))
|
|
73
|
+
rows.pop();
|
|
74
|
+
return rows;
|
|
75
|
+
}
|
|
76
|
+
const TRUE_WORDS = new Set(['yes', 'true', 'checked', 'done', 'y', '✓', '✔']);
|
|
77
|
+
const FALSE_WORDS = new Set(['no', 'false', 'unchecked', 'n', '✗', '✘']);
|
|
78
|
+
const RE_NUMBER = /^-?\d{1,3}(?:,\d{3})*(?:\.\d+)?$|^-?\d*\.?\d+$/;
|
|
79
|
+
const RE_CURRENCY = /^\s*([$£€¥₹﷼])\s*/;
|
|
80
|
+
const RE_EMAIL = /^[^\s@]+@[^\s@.]+\.[^\s@]+$/;
|
|
81
|
+
const RE_ISO_DATE = /^\d{4}-\d{2}-\d{2}(?:[T ]\d{2}:\d{2}(?::\d{2})?(?:\.\d+)?(?:Z|[+-]\d{2}:?\d{2})?)?$/;
|
|
82
|
+
const RE_LONG_DATE = /^[A-Za-z]{3,9}\s+\d{1,2},?\s+\d{4}(?:\s+\d{1,2}:\d{2}\s*(?:[AaPp][Mm])?)?$/;
|
|
83
|
+
const RE_SLASH_DATE = /^\d{1,2}\/\d{1,2}\/\d{4}$/;
|
|
84
|
+
/** A value nobody filled in. Notion writes an empty cell; a spreadsheet sometimes writes a dash. */
|
|
85
|
+
const isEmpty = (value) => value.trim() === '' || value.trim() === '-';
|
|
86
|
+
/** `2026-01-05`, `January 5, 2026`, `05/01/2026` — as the ISO string a date cell stores. */
|
|
87
|
+
export function parseDateValue(raw) {
|
|
88
|
+
const value = raw.trim();
|
|
89
|
+
if (RE_ISO_DATE.test(value)) {
|
|
90
|
+
const at = new Date(value.length === 10 ? `${value}T00:00:00Z` : value.replace(' ', 'T'));
|
|
91
|
+
return Number.isNaN(at.getTime()) ? null : value.length === 10 ? value : at.toISOString();
|
|
92
|
+
}
|
|
93
|
+
if (RE_LONG_DATE.test(value) || RE_SLASH_DATE.test(value)) {
|
|
94
|
+
const at = new Date(value);
|
|
95
|
+
if (Number.isNaN(at.getTime()))
|
|
96
|
+
return null;
|
|
97
|
+
// Date-only input has no time to keep, and a `date` cell draws the first ten characters.
|
|
98
|
+
return /\d{1,2}:\d{2}/.test(value) ? at.toISOString() : at.toISOString().slice(0, 10);
|
|
99
|
+
}
|
|
100
|
+
return null;
|
|
101
|
+
}
|
|
102
|
+
/** `1,234.50`, `$99`, `12%` — as the number a numeric sort and a rollup can use. */
|
|
103
|
+
export function parseNumberValue(raw) {
|
|
104
|
+
let value = raw.trim();
|
|
105
|
+
const currency = RE_CURRENCY.exec(value)?.[1] ?? null;
|
|
106
|
+
if (currency)
|
|
107
|
+
value = value.replace(RE_CURRENCY, '');
|
|
108
|
+
const percent = value.endsWith('%');
|
|
109
|
+
if (percent)
|
|
110
|
+
value = value.slice(0, -1).trim();
|
|
111
|
+
if (!RE_NUMBER.test(value))
|
|
112
|
+
return null;
|
|
113
|
+
const parsed = Number(value.replaceAll(',', ''));
|
|
114
|
+
return Number.isFinite(parsed) ? { value: parsed, percent, currency } : null;
|
|
115
|
+
}
|
|
116
|
+
/** An option id derived from its label, so the same choice keeps the same id across two imports. */
|
|
117
|
+
function optionId(label, taken) {
|
|
118
|
+
const base = label
|
|
119
|
+
.toLowerCase()
|
|
120
|
+
.normalize('NFKD')
|
|
121
|
+
.replace(/[^\p{Letter}\p{Number}]+/gu, '-')
|
|
122
|
+
.replace(/^-+|-+$/g, '')
|
|
123
|
+
.slice(0, 48) || 'option';
|
|
124
|
+
let id = base;
|
|
125
|
+
for (let n = 2; taken.has(id); n++)
|
|
126
|
+
id = `${base}-${n}`;
|
|
127
|
+
taken.add(id);
|
|
128
|
+
return id;
|
|
129
|
+
}
|
|
130
|
+
/**
|
|
131
|
+
* The palette a select column's choices are coloured from.
|
|
132
|
+
*
|
|
133
|
+
* Cycled rather than random: two imports of the same file produce the same colours, and a person
|
|
134
|
+
* re-running an import after fixing one row does not find their board repainted.
|
|
135
|
+
*/
|
|
136
|
+
const OPTION_COLOURS = ['slate', 'accent', 'success', 'warning', 'danger', 'info'];
|
|
137
|
+
/**
|
|
138
|
+
* One column's type, from its values.
|
|
139
|
+
*
|
|
140
|
+
* The order is the whole algorithm: checkbox before number (`1` and `0` are both), number before
|
|
141
|
+
* date (a bare year is a number), url and email before select (a column of six addresses is not six
|
|
142
|
+
* choices), multi-select before select (a comma is the only thing that separates them), and select
|
|
143
|
+
* before text (the fallback that is always true).
|
|
144
|
+
*/
|
|
145
|
+
export function guessColumn(name, rawValues) {
|
|
146
|
+
const values = rawValues.slice(0, SAMPLE).filter((value) => !isEmpty(value));
|
|
147
|
+
const total = values.length;
|
|
148
|
+
const from = `from ${total} of ${rawValues.length} value${rawValues.length === 1 ? '' : 's'}`;
|
|
149
|
+
const plain = (type, what, config = {}) => ({
|
|
150
|
+
name,
|
|
151
|
+
type,
|
|
152
|
+
config,
|
|
153
|
+
note: `${name}: read as ${what}, ${from}`,
|
|
154
|
+
});
|
|
155
|
+
if (total === 0)
|
|
156
|
+
return {
|
|
157
|
+
name,
|
|
158
|
+
type: 'text',
|
|
159
|
+
config: {},
|
|
160
|
+
note: `${name}: read as text — the column has no values to judge`,
|
|
161
|
+
};
|
|
162
|
+
const lower = values.map((value) => value.trim().toLowerCase());
|
|
163
|
+
if (lower.every((value) => TRUE_WORDS.has(value) || FALSE_WORDS.has(value)))
|
|
164
|
+
return plain('checkbox', 'a checkbox');
|
|
165
|
+
const numbers = values.map(parseNumberValue);
|
|
166
|
+
if (numbers.every((n) => n !== null)) {
|
|
167
|
+
const percent = numbers.every((n) => n.percent);
|
|
168
|
+
const currency = numbers.find((n) => n.currency)?.currency ?? null;
|
|
169
|
+
const decimals = values.map((value) => (value.split('.')[1] ?? '').replace(/[^\d]/g, '').length);
|
|
170
|
+
const config = {
|
|
171
|
+
format: percent ? 'percent' : currency ? 'currency' : 'plain',
|
|
172
|
+
precision: Math.min(8, Math.max(...decimals, 0)),
|
|
173
|
+
};
|
|
174
|
+
return plain('number', percent ? 'a percentage' : currency ? 'an amount of money' : 'a number', config);
|
|
175
|
+
}
|
|
176
|
+
if (values.every((value) => parseDateValue(value) !== null)) {
|
|
177
|
+
const withTime = values.some((value) => /\d{1,2}:\d{2}/.test(value));
|
|
178
|
+
return plain('date', withTime ? 'a date and time' : 'a date', { includeTime: withTime });
|
|
179
|
+
}
|
|
180
|
+
if (values.every((value) => /^https?:\/\/\S+$/i.test(value.trim())))
|
|
181
|
+
return plain('url', 'a link');
|
|
182
|
+
if (values.every((value) => RE_EMAIL.test(value.trim())))
|
|
183
|
+
return plain('email', 'an email address');
|
|
184
|
+
const options = (labels) => {
|
|
185
|
+
const taken = new Set();
|
|
186
|
+
const ids = new Map();
|
|
187
|
+
const out = [];
|
|
188
|
+
labels.forEach((label, index) => {
|
|
189
|
+
const id = optionId(label, taken);
|
|
190
|
+
ids.set(label, id);
|
|
191
|
+
out.push({ id, label, colour: OPTION_COLOURS[index % OPTION_COLOURS.length] });
|
|
192
|
+
});
|
|
193
|
+
return { options: out, ids };
|
|
194
|
+
};
|
|
195
|
+
// A multi-select is a select whose cells hold more than one choice, so it is only worth guessing
|
|
196
|
+
// when at least one cell actually does — otherwise every select with a comma in one label becomes
|
|
197
|
+
// one, and the column silently stops matching an `equals` filter.
|
|
198
|
+
if (values.some((value) => value.includes(','))) {
|
|
199
|
+
const parts = [...new Set(values.flatMap((value) => splitChoices(value)))];
|
|
200
|
+
if (parts.length > 0 && parts.length <= MAX_OPTIONS && parts.every((p) => p.length <= MAX_OPTION_LENGTH))
|
|
201
|
+
return {
|
|
202
|
+
name,
|
|
203
|
+
type: 'multi_select',
|
|
204
|
+
config: { options: options(parts).options },
|
|
205
|
+
note: `${name}: read as a multi-select with ${parts.length} choices, ${from}`,
|
|
206
|
+
};
|
|
207
|
+
}
|
|
208
|
+
const distinct = [...new Set(values.map((value) => value.trim()))];
|
|
209
|
+
if (distinct.length <= MAX_OPTIONS &&
|
|
210
|
+
distinct.length < total &&
|
|
211
|
+
distinct.every((value) => value.length <= MAX_OPTION_LENGTH))
|
|
212
|
+
return {
|
|
213
|
+
name,
|
|
214
|
+
type: 'select',
|
|
215
|
+
config: { options: options(distinct).options },
|
|
216
|
+
note: `${name}: read as a select with ${distinct.length} choices, ${from}`,
|
|
217
|
+
};
|
|
218
|
+
return plain('text', 'text');
|
|
219
|
+
}
|
|
220
|
+
/** `Design, Urgent` → `['Design', 'Urgent']`, which is how every exporter writes a multi-select. */
|
|
221
|
+
export const splitChoices = (value) => value
|
|
222
|
+
.split(',')
|
|
223
|
+
.map((part) => part.trim())
|
|
224
|
+
.filter((part) => part.length > 0);
|
|
225
|
+
/**
|
|
226
|
+
* One cell, as the value the column's type stores.
|
|
227
|
+
*
|
|
228
|
+
* Null means "leave the key out of `props`" rather than "write null": an absent key is what every
|
|
229
|
+
* other writer in this module produces for an empty cell, and a null would make a `not_contains`
|
|
230
|
+
* filter behave differently for an imported row than for one somebody typed.
|
|
231
|
+
*
|
|
232
|
+
* `select` stores a **scalar** option id and `multi_select` an **array** of them, and the difference
|
|
233
|
+
* is load-bearing rather than cosmetic: `query.ts` compares a select as text through `props->>'key'`,
|
|
234
|
+
* so an array there stringifies to `["done"]` and matches no filter anybody can write. The cell
|
|
235
|
+
* component reads either shape, which is exactly why this is easy to get wrong and invisible on screen.
|
|
236
|
+
*/
|
|
237
|
+
export function coerceValue(column, raw) {
|
|
238
|
+
if (isEmpty(raw))
|
|
239
|
+
return null;
|
|
240
|
+
const value = raw.trim();
|
|
241
|
+
switch (column.type) {
|
|
242
|
+
case 'checkbox':
|
|
243
|
+
return TRUE_WORDS.has(value.toLowerCase());
|
|
244
|
+
case 'number':
|
|
245
|
+
return parseNumberValue(value)?.value ?? null;
|
|
246
|
+
case 'date':
|
|
247
|
+
return parseDateValue(value);
|
|
248
|
+
case 'select': {
|
|
249
|
+
const option = (column.config.options ?? []).find((o) => o.label === value);
|
|
250
|
+
return option ? option.id : null;
|
|
251
|
+
}
|
|
252
|
+
case 'multi_select': {
|
|
253
|
+
const wanted = splitChoices(value);
|
|
254
|
+
const ids = wanted
|
|
255
|
+
.map((label) => (column.config.options ?? []).find((o) => o.label === label)?.id)
|
|
256
|
+
.filter((id) => id !== undefined);
|
|
257
|
+
return ids.length > 0 ? ids : null;
|
|
258
|
+
}
|
|
259
|
+
default:
|
|
260
|
+
return value;
|
|
261
|
+
}
|
|
262
|
+
}
|
|
263
|
+
//# sourceMappingURL=csv.js.map
|