@intentic/fileq 1.318.0 → 1.320.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/NOTICE +4 -4
- package/README.md +16 -10
- package/dist/app.d.ts.map +1 -1
- package/dist/app.js +4 -6
- package/dist/app.js.map +1 -1
- package/dist/commands/derive.command.js +1 -1
- package/dist/commands/derive.command.js.map +1 -1
- package/dist/commands/read.command.d.ts.map +1 -1
- package/dist/commands/read.command.js +26 -45
- package/dist/commands/read.command.js.map +1 -1
- package/dist/lib/check/{docx.d.ts → check-docx.d.ts} +1 -1
- package/dist/lib/check/check-docx.d.ts.map +1 -0
- package/dist/lib/check/{docx.js → check-docx.js} +46 -14
- package/dist/lib/check/check-docx.js.map +1 -0
- package/dist/lib/check/{pdf.d.ts → check-pdf.d.ts} +1 -1
- package/dist/lib/check/check-pdf.d.ts.map +1 -0
- package/dist/lib/check/{pdf.js → check-pdf.js} +4 -2
- package/dist/lib/check/check-pdf.js.map +1 -0
- package/dist/lib/check/{pptx.d.ts → check-pptx.d.ts} +1 -1
- package/dist/lib/check/check-pptx.d.ts.map +1 -0
- package/dist/lib/check/{pptx.js → check-pptx.js} +18 -5
- package/dist/lib/check/check-pptx.js.map +1 -0
- package/dist/lib/check/{xlsx.d.ts → check-xlsx.d.ts} +1 -1
- package/dist/lib/check/check-xlsx.d.ts.map +1 -0
- package/dist/lib/check/{xlsx.js → check-xlsx.js} +17 -6
- package/dist/lib/check/check-xlsx.js.map +1 -0
- package/dist/lib/check/check.js +4 -4
- package/dist/lib/check/check.js.map +1 -1
- package/dist/lib/content-cache.d.ts +12 -0
- package/dist/lib/content-cache.d.ts.map +1 -0
- package/dist/lib/content-cache.js +33 -0
- package/dist/lib/content-cache.js.map +1 -0
- package/dist/lib/derive.d.ts +10 -0
- package/dist/lib/derive.d.ts.map +1 -1
- package/dist/lib/derive.js +40 -4
- package/dist/lib/derive.js.map +1 -1
- package/dist/lib/sidecar.d.ts +11 -2
- package/dist/lib/sidecar.d.ts.map +1 -1
- package/dist/lib/sidecar.js +28 -16
- package/dist/lib/sidecar.js.map +1 -1
- package/dist/skill.d.ts.map +1 -1
- package/dist/skill.js +8 -7
- package/dist/skill.js.map +1 -1
- package/package.json +7 -7
- package/src/app.ts +4 -6
- package/src/cli.integration.test.ts +52 -29
- package/src/commands/check.command.ts +1 -1
- package/src/commands/derive.command.ts +1 -1
- package/src/commands/read.command.ts +30 -52
- package/src/lib/archives.ts +1 -1
- package/src/lib/check/{docx.ts → check-docx.ts} +112 -27
- package/src/lib/check/{pdf.ts → check-pdf.ts} +23 -4
- package/src/lib/check/{pptx.ts → check-pptx.ts} +48 -11
- package/src/lib/check/{xlsx.ts → check-xlsx.ts} +20 -6
- package/src/lib/check/check.test.ts +99 -27
- package/src/lib/check/check.ts +4 -4
- package/src/lib/content-cache.ts +58 -0
- package/src/lib/derive.integration.test.ts +198 -0
- package/src/lib/derive.ts +69 -7
- package/src/lib/derivers/archive.ts +1 -1
- package/src/lib/derivers/load-cost.test.ts +1 -1
- package/src/lib/derivers/pdf.ts +1 -1
- package/src/lib/derivers/xlsx.ts +1 -1
- package/src/lib/sidecar.ts +42 -22
- package/src/render.integration.test.ts +80 -56
- package/src/skill.ts +8 -7
- package/dist/commands/sweep.command.d.ts +0 -3
- package/dist/commands/sweep.command.d.ts.map +0 -1
- package/dist/commands/sweep.command.js +0 -41
- package/dist/commands/sweep.command.js.map +0 -1
- package/dist/lib/check/docx.d.ts.map +0 -1
- package/dist/lib/check/docx.js.map +0 -1
- package/dist/lib/check/pdf.d.ts.map +0 -1
- package/dist/lib/check/pdf.js.map +0 -1
- package/dist/lib/check/pptx.d.ts.map +0 -1
- package/dist/lib/check/pptx.js.map +0 -1
- package/dist/lib/check/xlsx.d.ts.map +0 -1
- package/dist/lib/check/xlsx.js.map +0 -1
- package/dist/lib/sweep.d.ts +0 -7
- package/dist/lib/sweep.d.ts.map +0 -1
- package/dist/lib/sweep.js +0 -56
- package/dist/lib/sweep.js.map +0 -1
- package/src/commands/sweep.command.ts +0 -48
- package/src/lib/sweep.ts +0 -72
package/src/app.ts
CHANGED
|
@@ -5,20 +5,19 @@ import { deriveCommand } from "./commands/derive.command.js";
|
|
|
5
5
|
import { gitAttributesCommand } from "./commands/git-attributes.command.js";
|
|
6
6
|
import { readCommand } from "./commands/read.command.js";
|
|
7
7
|
import { renderCommand } from "./commands/render.command.js";
|
|
8
|
-
import { sweepCommand } from "./commands/sweep.command.js";
|
|
9
8
|
|
|
10
9
|
// The agent-facing contract, kept small — this is what `fileq --help` prints.
|
|
11
10
|
const HELP = `fileq — binary workspace files as clean, budgeted markdown.
|
|
12
11
|
|
|
13
12
|
Reads the formats an agent cannot: docx, xlsx, pptx, pdf (text layer), images
|
|
14
|
-
(dimensions + EXIF), audio/video (duration + tags), html.
|
|
15
|
-
|
|
16
|
-
|
|
13
|
+
(dimensions + EXIF), audio/video (duration + tags), html. Renderings are cached
|
|
14
|
+
by content hash, so reading unchanged bytes again (the same file, a copy, a
|
|
15
|
+
download outside the workspace) is instant; each workspace file also gets a
|
|
16
|
+
markdown SIDECAR under .intentic/local/cache/derived/<path>.md.
|
|
17
17
|
|
|
18
18
|
fileq <file> print it as markdown (default: read), budgeted
|
|
19
19
|
fileq read <file> --budget 8000
|
|
20
20
|
fileq derive <file…> converge named files' sidecars (stale→derive, gone→remove)
|
|
21
|
-
fileq sweep converge the whole workspace, prune orphaned sidecars
|
|
22
21
|
fileq read --plain <f> the markdown alone, for a program; git's textconv driver
|
|
23
22
|
fileq git-attributes the gitattributes that route documents through it
|
|
24
23
|
|
|
@@ -47,7 +46,6 @@ export const app = buildApplication(
|
|
|
47
46
|
check: checkCommand,
|
|
48
47
|
render: renderCommand,
|
|
49
48
|
derive: deriveCommand,
|
|
50
|
-
sweep: sweepCommand,
|
|
51
49
|
gitAttributes: gitAttributesCommand,
|
|
52
50
|
},
|
|
53
51
|
defaultCommand: "read",
|
|
@@ -1,8 +1,8 @@
|
|
|
1
|
-
// CLI end-to-end against a temp workspace: derive, freshness, budgeted read,
|
|
2
|
-
//
|
|
1
|
+
// CLI end-to-end against a temp workspace: derive, freshness, the content cache, budgeted read, the ignore floor, and
|
|
2
|
+
// forged markers dying in the sidecar's bytes.
|
|
3
3
|
// Driven in-process through the same `run(app, …)` seam cli.ts calls (@intentic/agent-cli/testing); no build
|
|
4
4
|
// artifact, no child process.
|
|
5
|
-
import { existsSync, mkdirSync, mkdtempSync, readFileSync, rmSync, symlinkSync, writeFileSync } from "node:fs";
|
|
5
|
+
import { copyFileSync, existsSync, mkdirSync, mkdtempSync, readFileSync, rmSync, symlinkSync, writeFileSync } from "node:fs";
|
|
6
6
|
import { tmpdir } from "node:os";
|
|
7
7
|
import { dirname, join } from "node:path";
|
|
8
8
|
import { captureCli, type CliOutcome } from "@intentic/agent-cli/testing";
|
|
@@ -125,19 +125,16 @@ describe("read", () => {
|
|
|
125
125
|
expect(out).toContain("Paragraph 199 with a good number");
|
|
126
126
|
});
|
|
127
127
|
|
|
128
|
-
it("--plain on a file outside the workspace
|
|
128
|
+
it("--plain on a file outside the workspace keeps its rendering by content, so git's next textconv of that blob is cached", async () => {
|
|
129
129
|
const outside = mkdtempSync(join(tmpdir(), "fileq-outside-"));
|
|
130
|
-
const home = mkdtempSync(join(tmpdir(), "fileq-home-"));
|
|
131
|
-
process.env["FILEQ_HOME"] = home;
|
|
132
130
|
try {
|
|
133
|
-
writeFileSync(join(outside, "memo.docx"), docxBytes("Memo", ["
|
|
131
|
+
writeFileSync(join(outside, "memo.docx"), docxBytes("Memo", ["A blob git handed over."]));
|
|
134
132
|
const { out } = await fileq("read", "--plain", join(outside, "memo.docx"));
|
|
135
|
-
expect(out).toContain("
|
|
136
|
-
expect(
|
|
133
|
+
expect(out).toContain("A blob git handed over.");
|
|
134
|
+
expect(out).not.toContain("saved:");
|
|
135
|
+
expect(JSON.parse((await fileq("read", "--json", join(outside, "memo.docx"))).out)).toMatchObject({ source: "fresh" });
|
|
137
136
|
} finally {
|
|
138
|
-
delete process.env["FILEQ_HOME"];
|
|
139
137
|
rmSync(outside, { recursive: true, force: true });
|
|
140
|
-
rmSync(home, { recursive: true, force: true });
|
|
141
138
|
}
|
|
142
139
|
});
|
|
143
140
|
|
|
@@ -164,26 +161,52 @@ describe("read", () => {
|
|
|
164
161
|
});
|
|
165
162
|
});
|
|
166
163
|
|
|
167
|
-
describe("
|
|
168
|
-
|
|
169
|
-
|
|
170
|
-
|
|
171
|
-
|
|
172
|
-
|
|
173
|
-
|
|
174
|
-
|
|
175
|
-
expect(
|
|
176
|
-
expect(
|
|
177
|
-
|
|
178
|
-
|
|
179
|
-
|
|
180
|
-
|
|
164
|
+
describe("content cache", () => {
|
|
165
|
+
const readJson = async (path: string): Promise<{ source: string; path?: string }> => JSON.parse((await fileq("read", "--json", path)).out);
|
|
166
|
+
|
|
167
|
+
it("a copy of a workspace file at another path is fresh on its first read, with a sidecar of its own", async () => {
|
|
168
|
+
writeFileSync(join(root, "original.docx"), docxBytes("Original", ["Rendered once."]));
|
|
169
|
+
expect(await readJson(join(root, "original.docx"))).toMatchObject({ source: "derived" });
|
|
170
|
+
mkdirSync(join(root, "copies"), { recursive: true });
|
|
171
|
+
copyFileSync(join(root, "original.docx"), join(root, "copies/original.docx"));
|
|
172
|
+
expect(await readJson(join(root, "copies/original.docx"))).toMatchObject({ source: "fresh", path: sidecarOf("copies/original.docx") });
|
|
173
|
+
expect(readFileSync(sidecarOf("copies/original.docx"), "utf8")).toContain("source: copies/original.docx");
|
|
174
|
+
});
|
|
175
|
+
|
|
176
|
+
it("a file outside the workspace is derived once, then answered from the cache until its bytes change", async () => {
|
|
177
|
+
const outside = mkdtempSync(join(tmpdir(), "fileq-outside-"));
|
|
178
|
+
try {
|
|
179
|
+
writeFileSync(join(outside, "memo.docx"), docxBytes("Memo", ["First draft."]));
|
|
180
|
+
const first = await readJson(join(outside, "memo.docx"));
|
|
181
|
+
expect(first).toMatchObject({ source: "derived" });
|
|
182
|
+
expect(first.path?.startsWith(join(root, DERIVED_DIR, ".by-hash/"))).toBe(true);
|
|
183
|
+
expect(await readJson(join(outside, "memo.docx"))).toMatchObject({ source: "fresh", path: first.path });
|
|
184
|
+
writeFileSync(join(outside, "memo.docx"), docxBytes("Memo", ["Second draft."]));
|
|
185
|
+
const edited = await fileq("read", join(outside, "memo.docx"));
|
|
186
|
+
expect(edited.out).toContain("derived");
|
|
187
|
+
expect(edited.out).toContain("Second draft.");
|
|
188
|
+
} finally {
|
|
189
|
+
rmSync(outside, { recursive: true, force: true });
|
|
190
|
+
}
|
|
181
191
|
});
|
|
182
192
|
|
|
183
|
-
it("
|
|
184
|
-
const
|
|
185
|
-
|
|
186
|
-
|
|
193
|
+
it("with no workspace and an unwritable cache, the rendering is still printed, and says nothing was saved", async () => {
|
|
194
|
+
const outside = mkdtempSync(join(tmpdir(), "fileq-outside-"));
|
|
195
|
+
delete process.env["WORKSPACE_ROOT"];
|
|
196
|
+
process.env["FILEQ_HOME"] = join(outside, "blocker");
|
|
197
|
+
try {
|
|
198
|
+
writeFileSync(join(outside, "blocker"), "a file where the cache directory would go");
|
|
199
|
+
writeFileSync(join(outside, "memo.docx"), docxBytes("Memo", ["Still readable."]));
|
|
200
|
+
const { out, exitCode } = await fileq("read", join(outside, "memo.docx"));
|
|
201
|
+
expect(exitCode).toBe(0);
|
|
202
|
+
expect(out).toContain("Still readable.");
|
|
203
|
+
expect(out).toContain("note: nothing saved: the rendering cache could not be written");
|
|
204
|
+
expect(out).not.toMatch(/^saved: /m);
|
|
205
|
+
} finally {
|
|
206
|
+
process.env["WORKSPACE_ROOT"] = root;
|
|
207
|
+
delete process.env["FILEQ_HOME"];
|
|
208
|
+
rmSync(outside, { recursive: true, force: true });
|
|
209
|
+
}
|
|
187
210
|
});
|
|
188
211
|
});
|
|
189
212
|
|
|
@@ -62,7 +62,7 @@ export const checkCommand = buildCommand({
|
|
|
62
62
|
},
|
|
63
63
|
async func(this: CommandContext, flags: CheckFlags, file: string) {
|
|
64
64
|
const absPath = resolve(file);
|
|
65
|
-
// Loaded here, not at the top: every fileq run (a
|
|
65
|
+
// Loaded here, not at the top: every fileq run (a derive, a read of a png) imports this module, and only a check
|
|
66
66
|
// needs the checkers.
|
|
67
67
|
const { CHECKABLE, checkFile } = await import("../lib/check/check.js");
|
|
68
68
|
const outcome = await checkFile(absPath);
|
|
@@ -37,7 +37,7 @@ export const deriveCommand = buildCommand({
|
|
|
37
37
|
return;
|
|
38
38
|
}
|
|
39
39
|
if (files.length === 0) {
|
|
40
|
-
this.process.stdout.write("fileq: derive takes file paths
|
|
40
|
+
this.process.stdout.write("fileq: derive takes file paths\n");
|
|
41
41
|
process.exitCode = 2;
|
|
42
42
|
return;
|
|
43
43
|
}
|
|
@@ -1,16 +1,11 @@
|
|
|
1
1
|
/* `fileq read <file>` (also the default command): one file as markdown — a capsule line saying what happened, the content up to a token budget; `--plain` is the body alone, whole, for a program (git's textconv) rather than an agent. */
|
|
2
|
-
import {
|
|
3
|
-
import { mkdir, writeFile } from "node:fs/promises";
|
|
4
|
-
import { basename, join, resolve } from "node:path";
|
|
5
|
-
import { toolOutDir } from "@intentic/agent-cli/env";
|
|
2
|
+
import { basename, resolve } from "node:path";
|
|
6
3
|
import { countParser } from "@intentic/agent-cli/flags";
|
|
7
4
|
import { capsule, clip } from "@intentic/agent-cli/output";
|
|
8
5
|
import { estimateTokens } from "@intentic/base/format";
|
|
9
6
|
import { buildCommand, type CommandContext } from "@stricli/core";
|
|
10
|
-
import {
|
|
11
|
-
import {
|
|
12
|
-
import { detectFormat } from "../lib/formats.js";
|
|
13
|
-
import { DERIVERS, ensureSidecar, type Outcome } from "../lib/derive.js";
|
|
7
|
+
import { deriverStamp } from "../lib/derivers/deriver.js";
|
|
8
|
+
import { DERIVERS, ensureSidecar, renderByContent, type Outcome } from "../lib/derive.js";
|
|
14
9
|
import { workspaceRoot } from "../lib/env.js";
|
|
15
10
|
|
|
16
11
|
interface ReadFlags {
|
|
@@ -25,7 +20,7 @@ export const readCommand = buildCommand({
|
|
|
25
20
|
flags: {
|
|
26
21
|
budget: { kind: "parsed", parse: countParser, default: "4000", brief: "Max stdout tokens; 0 prints only the capsule" },
|
|
27
22
|
json: { kind: "boolean", default: false, brief: "Machine-readable result on stdout" },
|
|
28
|
-
plain: { kind: "boolean", default: false, brief: "The markdown alone, whole and unbudgeted
|
|
23
|
+
plain: { kind: "boolean", default: false, brief: "The markdown alone, whole and unbudgeted (git textconv)" },
|
|
29
24
|
},
|
|
30
25
|
positional: {
|
|
31
26
|
kind: "tuple",
|
|
@@ -35,8 +30,7 @@ export const readCommand = buildCommand({
|
|
|
35
30
|
async func(this: CommandContext, flags: ReadFlags, file: string) {
|
|
36
31
|
const absPath = resolve(file);
|
|
37
32
|
const root = workspaceRoot();
|
|
38
|
-
const
|
|
39
|
-
const result = root === undefined ? await readOutsideWorkspace(absPath, save) : await readInWorkspace(root, absPath, save);
|
|
33
|
+
const result = root === undefined ? await readOutsideWorkspace(undefined, absPath) : await readInWorkspace(root, absPath);
|
|
40
34
|
if (result === undefined) {
|
|
41
35
|
process.exitCode = 1;
|
|
42
36
|
return;
|
|
@@ -52,11 +46,15 @@ export const readCommand = buildCommand({
|
|
|
52
46
|
return;
|
|
53
47
|
}
|
|
54
48
|
const fields = [result.title ?? basename(absPath), result.format, `${result.tokens} tokens`, result.source];
|
|
55
|
-
|
|
56
|
-
this.process.stdout.write(
|
|
49
|
+
const notes = result.savedPath === undefined ? [...result.notes, "nothing saved: the rendering cache could not be written"] : result.notes;
|
|
50
|
+
this.process.stdout.write(capsule("fileq", fields, notes));
|
|
51
|
+
if (result.savedPath !== undefined) {
|
|
52
|
+
this.process.stdout.write(`saved: ${result.savedPath}\n`);
|
|
53
|
+
}
|
|
57
54
|
if (flags.budget > 0 && result.body !== "") {
|
|
55
|
+
const whole = result.savedPath ?? `the output of \`fileq read --plain ${absPath}\``;
|
|
58
56
|
this.process.stdout.write("---\n");
|
|
59
|
-
this.process.stdout.write(withFinalNewline(clip(result.body, flags.budget,
|
|
57
|
+
this.process.stdout.write(withFinalNewline(clip(result.body, flags.budget, whole, "document")));
|
|
60
58
|
}
|
|
61
59
|
},
|
|
62
60
|
});
|
|
@@ -67,16 +65,17 @@ interface ReadResult {
|
|
|
67
65
|
readonly deriver: string;
|
|
68
66
|
readonly body: string;
|
|
69
67
|
readonly tokens: number;
|
|
70
|
-
|
|
68
|
+
/** The file holding the whole rendering; undefined when the cache could not be written. */
|
|
69
|
+
readonly savedPath: string | undefined;
|
|
71
70
|
readonly source: "derived" | "fresh";
|
|
72
71
|
readonly title?: string | undefined;
|
|
73
72
|
readonly notes: string[];
|
|
74
73
|
}
|
|
75
74
|
|
|
76
|
-
const readInWorkspace = async (root: string, absPath: string
|
|
75
|
+
const readInWorkspace = async (root: string, absPath: string): Promise<ReadResult | undefined> => {
|
|
77
76
|
const outcome = await ensureSidecar(root, absPath);
|
|
78
77
|
if (outcome.kind === "skipped" && outcome.reason === "outside-workspace") {
|
|
79
|
-
return readOutsideWorkspace(
|
|
78
|
+
return readOutsideWorkspace(root, absPath);
|
|
80
79
|
}
|
|
81
80
|
return fromOutcome(outcome);
|
|
82
81
|
};
|
|
@@ -113,44 +112,23 @@ const fromOutcome = (outcome: Outcome): ReadResult | undefined => {
|
|
|
113
112
|
}
|
|
114
113
|
};
|
|
115
114
|
|
|
116
|
-
|
|
117
|
-
|
|
118
|
-
const readOutsideWorkspace = async (
|
|
119
|
-
const
|
|
120
|
-
if (
|
|
121
|
-
process.stdout.write(`fileq: cannot read ${absPath}:
|
|
122
|
-
return undefined;
|
|
123
|
-
}
|
|
124
|
-
// The same outcome a corrupt file gets inside a workspace (ensureSidecar's loud skip): a notebook that is
|
|
125
|
-
// not JSON, a docx that is not a zip, answers with the reason and exit 1, never a stack trace.
|
|
126
|
-
let doc: DerivedDoc;
|
|
127
|
-
try {
|
|
128
|
-
doc = neutralizeDoc(await DERIVERS[format].derive(absPath));
|
|
129
|
-
} catch (error) {
|
|
130
|
-
process.stdout.write(`fileq: cannot read ${absPath}: derive-failed (${format}): ${errorMessage(error).split("\n")[0]}\n`);
|
|
115
|
+
// A file outside the workspace has no sidecar: the content cache alone keeps its rendering, and its entry is what `saved:`
|
|
116
|
+
// names, so git's textconv reading the same blob twice derives it once.
|
|
117
|
+
const readOutsideWorkspace = async (root: string | undefined, absPath: string): Promise<ReadResult | undefined> => {
|
|
118
|
+
const outcome = await renderByContent(root, absPath);
|
|
119
|
+
if (outcome.kind === "failed") {
|
|
120
|
+
process.stdout.write(`fileq: cannot read ${absPath}: ${outcome.reason}\n`);
|
|
131
121
|
return undefined;
|
|
132
122
|
}
|
|
133
|
-
const outDir = toolOutDir("fileq");
|
|
134
|
-
const hash = createHash("sha256").update(absPath).digest("hex").slice(0, 8);
|
|
135
|
-
const savedPath = join(
|
|
136
|
-
outDir,
|
|
137
|
-
`${basename(absPath)
|
|
138
|
-
.toLowerCase()
|
|
139
|
-
.replaceAll(/[^a-z0-9.]+/g, "-")}-${hash}.md`,
|
|
140
|
-
);
|
|
141
|
-
if (save) {
|
|
142
|
-
await mkdir(outDir, { recursive: true });
|
|
143
|
-
await writeFile(savedPath, `${doc.markdown}\n`);
|
|
144
|
-
}
|
|
145
123
|
return {
|
|
146
|
-
format,
|
|
147
|
-
deriver: deriverStamp(DERIVERS[format]),
|
|
148
|
-
body: doc.markdown,
|
|
149
|
-
tokens: estimateTokens(doc.markdown),
|
|
150
|
-
savedPath,
|
|
151
|
-
source:
|
|
152
|
-
title: doc.title,
|
|
153
|
-
notes: doc.notes,
|
|
124
|
+
format: outcome.format,
|
|
125
|
+
deriver: deriverStamp(DERIVERS[outcome.format]),
|
|
126
|
+
body: outcome.doc.markdown,
|
|
127
|
+
tokens: estimateTokens(outcome.doc.markdown),
|
|
128
|
+
savedPath: outcome.cachedPath,
|
|
129
|
+
source: outcome.kind,
|
|
130
|
+
title: outcome.doc.title,
|
|
131
|
+
notes: outcome.doc.notes,
|
|
154
132
|
};
|
|
155
133
|
};
|
|
156
134
|
|
package/src/lib/archives.ts
CHANGED
|
@@ -198,7 +198,7 @@ export const parseTarListing = (stdout: string): ArchiveEntry[] => {
|
|
|
198
198
|
return entries;
|
|
199
199
|
};
|
|
200
200
|
|
|
201
|
-
// Long enough for a large archive's index, short enough that a hung tar cannot hold
|
|
201
|
+
// Long enough for a large archive's index, short enough that a hung tar cannot hold a read hostage.
|
|
202
202
|
const TAR_TIMEOUT_MS = 60_000;
|
|
203
203
|
const TAR_MAX_BUFFER = 16 * 1024 * 1024;
|
|
204
204
|
|
|
@@ -73,10 +73,12 @@ const headingLevels = (styles: XmlElement | undefined): ReadonlyMap<string, numb
|
|
|
73
73
|
const basedOn = childNamed(style, "w:basedOn")?.attrs["w:val"];
|
|
74
74
|
return basedOn === undefined ? undefined : levelOf(basedOn, depth + 1);
|
|
75
75
|
};
|
|
76
|
-
return new Map(
|
|
77
|
-
|
|
78
|
-
|
|
79
|
-
|
|
76
|
+
return new Map(
|
|
77
|
+
[...byId.keys()].flatMap((id) => {
|
|
78
|
+
const level = levelOf(id, 0);
|
|
79
|
+
return level === undefined ? [] : [[id, level] as const];
|
|
80
|
+
}),
|
|
81
|
+
);
|
|
80
82
|
};
|
|
81
83
|
|
|
82
84
|
const headingOf = (paragraph: XmlElement, levels: ReadonlyMap<string, number>): number | undefined => {
|
|
@@ -106,7 +108,12 @@ const locate = (paragraphs: readonly XmlElement[], levels: ReadonlyMap<string, n
|
|
|
106
108
|
const text = paragraphText(paragraph);
|
|
107
109
|
const heading = headingOf(paragraph, levels);
|
|
108
110
|
const number = index + 1;
|
|
109
|
-
const label =
|
|
111
|
+
const label =
|
|
112
|
+
text.trim() !== ""
|
|
113
|
+
? `paragraph ${number} · ${excerpt(text, 40)}`
|
|
114
|
+
: lastHeading === undefined
|
|
115
|
+
? `paragraph ${number}`
|
|
116
|
+
: `paragraph ${number} · under ${lastHeading}`;
|
|
110
117
|
located.push({ paragraph, number, text, heading });
|
|
111
118
|
whereOf.set(paragraph, label);
|
|
112
119
|
if (heading !== undefined && text.trim() !== "") {
|
|
@@ -151,7 +158,12 @@ const stepField = (node: XmlElement, open: OpenField[], fields: Field[], where:
|
|
|
151
158
|
current.separated = true;
|
|
152
159
|
} else if (type === "end") {
|
|
153
160
|
open.pop();
|
|
154
|
-
fields.push({
|
|
161
|
+
fields.push({
|
|
162
|
+
instruction: current.instruction.trim(),
|
|
163
|
+
result: current.result,
|
|
164
|
+
hasResult: current.separated && current.result.trim() !== "",
|
|
165
|
+
where: current.where,
|
|
166
|
+
});
|
|
155
167
|
}
|
|
156
168
|
};
|
|
157
169
|
|
|
@@ -195,19 +207,33 @@ const bookmarkArgument = (instruction: string): string | undefined => {
|
|
|
195
207
|
const fieldFindings = (fields: readonly Field[], bookmarks: ReadonlySet<string>, updatesOnOpen: boolean): Finding[] =>
|
|
196
208
|
fields.flatMap((field): Finding[] => {
|
|
197
209
|
const keyword = (field.instruction.split(/\s+/)[0] ?? "").toUpperCase();
|
|
198
|
-
if (field.result.trim().startsWith(
|
|
199
|
-
return [
|
|
210
|
+
if (field.result.trim().startsWith("Error!")) {
|
|
211
|
+
return [
|
|
212
|
+
error("field-error", field.where, `the ${keyword} field shows ${excerpt(field.result)}: fix what it points at, then update fields`),
|
|
213
|
+
];
|
|
200
214
|
}
|
|
201
215
|
const bookmark = bookmarkArgument(field.instruction);
|
|
202
216
|
if (bookmark !== undefined && !bookmarks.has(bookmark) && !BUILT_IN_ANCHORS.has(bookmark)) {
|
|
203
|
-
return [
|
|
217
|
+
return [
|
|
218
|
+
error(
|
|
219
|
+
"broken-reference",
|
|
220
|
+
field.where,
|
|
221
|
+
`the ${keyword} field points to bookmark "${bookmark}", which is not in the document: Word shows "Error! Reference source not found."`,
|
|
222
|
+
),
|
|
223
|
+
];
|
|
204
224
|
}
|
|
205
225
|
if (keyword === "MERGEFIELD") {
|
|
206
|
-
return [
|
|
226
|
+
return [
|
|
227
|
+
warning("merge-field", field.where, `a mail-merge field is left in (${field.instruction}): it prints its placeholder, not a value`),
|
|
228
|
+
];
|
|
207
229
|
}
|
|
208
230
|
if (keyword === "TOC" && !field.hasResult) {
|
|
209
|
-
const when = updatesOnOpen
|
|
210
|
-
|
|
231
|
+
const when = updatesOnOpen
|
|
232
|
+
? "Word offers to update fields when the file opens"
|
|
233
|
+
: "nothing fills it until someone updates fields (F9 in Word)";
|
|
234
|
+
return [
|
|
235
|
+
warning("empty-toc", field.where, `the table of contents has no entries saved: ${when}, and previews and LibreOffice show it empty`),
|
|
236
|
+
];
|
|
211
237
|
}
|
|
212
238
|
if (!field.hasResult && keyword !== "" && !COMPUTED_FIELDS.has(keyword) && keyword !== "TOC") {
|
|
213
239
|
return [warning("empty-field", field.where, `the ${keyword} field has no saved result, so it shows blank until fields are updated`)];
|
|
@@ -217,7 +243,13 @@ const fieldFindings = (fields: readonly Field[], bookmarks: ReadonlySet<string>,
|
|
|
217
243
|
|
|
218
244
|
// --- references and images -------------------------------------------------------------------------------------
|
|
219
245
|
|
|
220
|
-
const referenceFindings = (
|
|
246
|
+
const referenceFindings = (
|
|
247
|
+
context: DocContext,
|
|
248
|
+
part: string,
|
|
249
|
+
tree: XmlElement,
|
|
250
|
+
whereOfNode: (node: XmlElement) => string,
|
|
251
|
+
referenced: Set<string>,
|
|
252
|
+
): Finding[] => {
|
|
221
253
|
const rels = context.pkg.rels(part);
|
|
222
254
|
const findings: Finding[] = [];
|
|
223
255
|
for (const node of everyElement(tree)) {
|
|
@@ -244,10 +276,20 @@ const referenceFinding = (
|
|
|
244
276
|
): Finding | undefined => {
|
|
245
277
|
const isImage = node.name === "a:blip" || node.name === "v:imagedata";
|
|
246
278
|
if (relationship === undefined) {
|
|
247
|
-
return error(
|
|
279
|
+
return error(
|
|
280
|
+
isImage ? "missing-image" : "missing-part",
|
|
281
|
+
where,
|
|
282
|
+
`${isImage ? "an image" : node.name} refers to ${id}, which the document does not define: it shows as missing`,
|
|
283
|
+
);
|
|
248
284
|
}
|
|
249
285
|
if (relationship.external) {
|
|
250
|
-
return isImage
|
|
286
|
+
return isImage
|
|
287
|
+
? warning(
|
|
288
|
+
"linked-image",
|
|
289
|
+
where,
|
|
290
|
+
`an image is linked from outside the file (${relationship.target}), so it will not show on anyone else's machine; embed it`,
|
|
291
|
+
)
|
|
292
|
+
: undefined;
|
|
251
293
|
}
|
|
252
294
|
if (pkg.has(relationship.target)) {
|
|
253
295
|
return undefined;
|
|
@@ -290,7 +332,13 @@ const wideImageFindings = (context: DocContext, body: XmlElement, owner: Readonl
|
|
|
290
332
|
return [];
|
|
291
333
|
}
|
|
292
334
|
const where = paragraph === undefined ? "" : (context.whereOf.get(paragraph) ?? "");
|
|
293
|
-
return [
|
|
335
|
+
return [
|
|
336
|
+
warning(
|
|
337
|
+
"wide-image",
|
|
338
|
+
where,
|
|
339
|
+
`a picture is ${inches(cx)} wide in a ${inches(width)} text column, so it runs into the margin or is cut off; scale it to fit`,
|
|
340
|
+
),
|
|
341
|
+
];
|
|
294
342
|
});
|
|
295
343
|
};
|
|
296
344
|
|
|
@@ -298,11 +346,17 @@ const wideImageFindings = (context: DocContext, body: XmlElement, owner: Readonl
|
|
|
298
346
|
|
|
299
347
|
const draftFindings = (context: DocContext, body: XmlElement): Finding[] => {
|
|
300
348
|
const findings: Finding[] = [];
|
|
301
|
-
const changes = everyElement(body).filter(
|
|
349
|
+
const changes = everyElement(body).filter(
|
|
350
|
+
(node) => ["w:ins", "w:del", "w:moveFrom", "w:moveTo"].includes(node.name) && node.children.some(isElement),
|
|
351
|
+
).length;
|
|
302
352
|
if (changes > 0) {
|
|
303
|
-
findings.push(
|
|
353
|
+
findings.push(
|
|
354
|
+
warning("tracked-changes", "", `${plural(changes, "tracked change")} still in the document: accept or reject them before it goes out`),
|
|
355
|
+
);
|
|
304
356
|
}
|
|
305
|
-
const commentsPart = [...context.pkg.rels(context.main).values()].find(
|
|
357
|
+
const commentsPart = [...context.pkg.rels(context.main).values()].find(
|
|
358
|
+
(relationship) => relationship.kind === "comments" && !relationship.external,
|
|
359
|
+
);
|
|
306
360
|
const comments = commentsPart === undefined ? undefined : context.pkg.xml(commentsPart.target);
|
|
307
361
|
const count = comments === undefined ? 0 : childrenNamed(comments, "w:comment").length;
|
|
308
362
|
if (count > 0) {
|
|
@@ -321,7 +375,13 @@ const paragraphFindings = (context: DocContext): Finding[] => {
|
|
|
321
375
|
const hasPicture = descendantsNamed(located.paragraph, "w:drawing").length > 0 || descendantsNamed(located.paragraph, "w:pict").length > 0;
|
|
322
376
|
if (located.heading !== undefined && located.text.trim() === "" && !hasPicture) {
|
|
323
377
|
const level = located.heading === 0 ? "title" : `heading ${located.heading}`;
|
|
324
|
-
findings.push(
|
|
378
|
+
findings.push(
|
|
379
|
+
warning(
|
|
380
|
+
"empty-heading",
|
|
381
|
+
where,
|
|
382
|
+
`empty ${level}: a blank line in the text and an empty entry in the navigation pane and any table of contents; delete it`,
|
|
383
|
+
),
|
|
384
|
+
);
|
|
325
385
|
}
|
|
326
386
|
if (TYPED_BULLET.test(located.text) && childAt(located.paragraph, "w:pPr", "w:numPr") === undefined) {
|
|
327
387
|
typedBullets += 1;
|
|
@@ -330,7 +390,11 @@ const paragraphFindings = (context: DocContext): Finding[] => {
|
|
|
330
390
|
}
|
|
331
391
|
if (firstBullet !== undefined) {
|
|
332
392
|
findings.push(
|
|
333
|
-
warning(
|
|
393
|
+
warning(
|
|
394
|
+
"typed-bullets",
|
|
395
|
+
firstBullet,
|
|
396
|
+
`${plural(typedBullets, "paragraph")} ${typedBullets === 1 ? "starts" : "start"} with a typed bullet character instead of a list style, so the list will not indent or renumber; use a list style`,
|
|
397
|
+
),
|
|
334
398
|
);
|
|
335
399
|
}
|
|
336
400
|
return findings;
|
|
@@ -343,7 +407,11 @@ const controlFindings = (body: XmlElement, whereOfNode: (node: XmlElement) => st
|
|
|
343
407
|
const prompt = textIn(childNamed(control, "w:sdtContent") ?? control, "w:t");
|
|
344
408
|
// A block-level control holds whole paragraphs; it is found by the first of them.
|
|
345
409
|
const where = whereOfNode(descendantsNamed(control, "w:p")[0] ?? control);
|
|
346
|
-
return error(
|
|
410
|
+
return error(
|
|
411
|
+
"placeholder-control",
|
|
412
|
+
where,
|
|
413
|
+
`a content control still shows its prompt ${excerpt(prompt)}: enter the value or remove the control`,
|
|
414
|
+
);
|
|
347
415
|
});
|
|
348
416
|
|
|
349
417
|
const linkFindings = (body: XmlElement, bookmarks: ReadonlySet<string>, whereOfNode: (node: XmlElement) => string): Finding[] =>
|
|
@@ -352,7 +420,13 @@ const linkFindings = (body: XmlElement, bookmarks: ReadonlySet<string>, whereOfN
|
|
|
352
420
|
if (anchor === undefined || bookmarks.has(anchor) || BUILT_IN_ANCHORS.has(anchor)) {
|
|
353
421
|
return [];
|
|
354
422
|
}
|
|
355
|
-
return [
|
|
423
|
+
return [
|
|
424
|
+
error(
|
|
425
|
+
"broken-link",
|
|
426
|
+
whereOfNode(link),
|
|
427
|
+
`the link ${excerpt(textIn(link, "w:t"))} points to bookmark "${anchor}", which is not in the document, so clicking it goes nowhere`,
|
|
428
|
+
),
|
|
429
|
+
];
|
|
356
430
|
});
|
|
357
431
|
|
|
358
432
|
// Headers, footers and notes: each part's own images and relationships, located by the part's name.
|
|
@@ -360,7 +434,10 @@ const storyFindings = (context: DocContext, stories: readonly { part: string; tr
|
|
|
360
434
|
stories.flatMap((story) => {
|
|
361
435
|
const label = story.part.replace(/^word\//, "").replace(/\.xml$/, "");
|
|
362
436
|
const referenced = new Set<string>();
|
|
363
|
-
return [
|
|
437
|
+
return [
|
|
438
|
+
...referenceFindings(context, story.part, story.tree, () => label, referenced),
|
|
439
|
+
...missingTargetFindings(context.pkg, story.part, label, referenced),
|
|
440
|
+
];
|
|
364
441
|
});
|
|
365
442
|
|
|
366
443
|
const openDocx = (bytes: Uint8Array): OoxmlPackage | CheckReport => {
|
|
@@ -368,7 +445,11 @@ const openDocx = (bytes: Uint8Array): OoxmlPackage | CheckReport => {
|
|
|
368
445
|
return openPackage(bytes);
|
|
369
446
|
} catch (cause) {
|
|
370
447
|
if (cause instanceof NotAPackage) {
|
|
371
|
-
return {
|
|
448
|
+
return {
|
|
449
|
+
format: "docx",
|
|
450
|
+
findings: [error("unreadable", "", `${cause.message}: this is not a Word file (or it is password-protected)`)],
|
|
451
|
+
notes: [],
|
|
452
|
+
};
|
|
372
453
|
}
|
|
373
454
|
throw cause;
|
|
374
455
|
}
|
|
@@ -403,7 +484,9 @@ export const checkDocx = (bytes: Uint8Array): CheckReport => {
|
|
|
403
484
|
return tree === undefined ? [] : [{ part: relationship.target, tree }];
|
|
404
485
|
});
|
|
405
486
|
const bookmarks = new Set(
|
|
406
|
-
[body, ...stories.map((story) => story.tree)].flatMap((tree) =>
|
|
487
|
+
[body, ...stories.map((story) => story.tree)].flatMap((tree) =>
|
|
488
|
+
descendantsNamed(tree, "w:bookmarkStart").flatMap((mark) => mark.attrs["w:name"] ?? []),
|
|
489
|
+
),
|
|
407
490
|
);
|
|
408
491
|
const context: DocContext = { pkg, main, paragraphs: located, whereOf, bookmarks };
|
|
409
492
|
if (located.every((item) => item.text.trim() === "") && descendantsNamed(body, "w:drawing").length === 0) {
|
|
@@ -424,7 +507,9 @@ export const checkDocx = (bytes: Uint8Array): CheckReport => {
|
|
|
424
507
|
);
|
|
425
508
|
// A field showing "Error!" or a control showing its prompt is already named for what it is; the same words read as
|
|
426
509
|
// leftover text would say it twice.
|
|
427
|
-
const explained = new Set(
|
|
510
|
+
const explained = new Set(
|
|
511
|
+
findings.filter((finding) => finding.rule === "field-error" || finding.rule === "placeholder-control").map((finding) => finding.where),
|
|
512
|
+
);
|
|
428
513
|
return {
|
|
429
514
|
format: "docx",
|
|
430
515
|
extent: plural(located.length, "paragraph"),
|
|
@@ -157,7 +157,13 @@ const linkFindings = async (pdf: PdfDocument, page: PdfPage, number: number): Pr
|
|
|
157
157
|
// allow(silent-catch): pdf.js throws where the file's destination tree is damaged, and then the link goes nowhere too.
|
|
158
158
|
const target = await pdf.getDestination(named).catch(() => null);
|
|
159
159
|
if (target === null) {
|
|
160
|
-
findings.push(
|
|
160
|
+
findings.push(
|
|
161
|
+
error(
|
|
162
|
+
"broken-link",
|
|
163
|
+
`page ${number}`,
|
|
164
|
+
`a link goes to "${named}", a destination the file does not define, so clicking it does nothing`,
|
|
165
|
+
),
|
|
166
|
+
);
|
|
161
167
|
}
|
|
162
168
|
}
|
|
163
169
|
return findings;
|
|
@@ -173,7 +179,10 @@ const openPdf = async (bytes: Uint8Array): Promise<PdfDocument | CheckReport> =>
|
|
|
173
179
|
// Errors only: pdf.js otherwise narrates its recovery of a damaged file on the console, into the report.
|
|
174
180
|
return await getDocumentProxy(bytes, { verbosity: 0 });
|
|
175
181
|
} catch (cause) {
|
|
176
|
-
const reason =
|
|
182
|
+
const reason =
|
|
183
|
+
cause instanceof Error && cause.name === "PasswordException"
|
|
184
|
+
? "it is password-protected, so nothing past the password can be checked"
|
|
185
|
+
: `it is not a readable PDF (${cause instanceof Error ? cause.message : String(cause)})`;
|
|
177
186
|
return { format: "pdf", findings: [error("unreadable", "", reason)], notes: [] };
|
|
178
187
|
}
|
|
179
188
|
};
|
|
@@ -211,14 +220,24 @@ export const checkPdf = async (bytes: Uint8Array): Promise<CheckReport> => {
|
|
|
211
220
|
}
|
|
212
221
|
for (const [font, pages] of unembedded) {
|
|
213
222
|
findings.push(
|
|
214
|
-
warning(
|
|
223
|
+
warning(
|
|
224
|
+
"font-not-embedded",
|
|
225
|
+
`page ${pageList(pages)}`,
|
|
226
|
+
`text is set in ${font}, which the file does not carry: a reader without that font substitutes another, and lines can break differently; embed it`,
|
|
227
|
+
),
|
|
215
228
|
);
|
|
216
229
|
}
|
|
217
230
|
if (sizes.size > 1) {
|
|
218
231
|
const [common] = [...sizes.entries()].toSorted((a, b) => b[1].length - a[1].length);
|
|
219
232
|
const odd = [...sizes.entries()].filter(([label]) => label !== common?.[0]);
|
|
220
233
|
for (const [label, pages] of odd) {
|
|
221
|
-
findings.push(
|
|
234
|
+
findings.push(
|
|
235
|
+
warning(
|
|
236
|
+
"page-size",
|
|
237
|
+
`page ${pageList(pages)}`,
|
|
238
|
+
`${plural(pages.length, "page is", "pages are")} ${label}, the rest ${common?.[0] ?? "another size"}`,
|
|
239
|
+
),
|
|
240
|
+
);
|
|
222
241
|
}
|
|
223
242
|
}
|
|
224
243
|
if (characters === 0 && total > 0) {
|