@zmskco/dsh-notebook 0.2.0 → 0.3.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/lib/http/api.js CHANGED
@@ -99,7 +99,11 @@ export function registerApi(deps) {
99
99
  }
100
100
  case 'files': {
101
101
  const { store, project } = await open(url.searchParams.get('project'));
102
- return json({ project: project.id, files: await store.listFiles(project) });
102
+ // `truncated` travels with the listing: the tree stops at an entry cap,
103
+ // and a capped list reported as a complete one would have the panel show
104
+ // a folder as empty when it is only large.
105
+ const listing = await store.listFiles(project);
106
+ return json({ project: project.id, files: listing.files, truncated: listing.truncated });
103
107
  }
104
108
  case 'file': {
105
109
  const { store, project } = await open(url.searchParams.get('project'));
@@ -2,34 +2,157 @@
2
2
  * What the notebook's tree shows, and how a file is read back.
3
3
  *
4
4
  * The format layer, kept apart from the IO that serves it: everything here is
5
- * pure — which names the browser may open, how a text is counted — and the store
5
+ * pure — which names the editor may open, how a text is counted — and the store
6
6
  * in `project.ts` owns the filesystem, the sandbox policy and the listing.
7
7
  *
8
+ * **The tree lists everything under the directory.** It used to run an extension
9
+ * allowlist (`md`, `txt`, `json`, …), which hid the code an author works next to and
10
+ * then, after that was widened, still hid every binary — and a tree that hides a
11
+ * file the filesystem has is a tree the user cannot trust: "my file is not there"
12
+ * is the report it produces. So every regular file is listed, and the two questions
13
+ * are answered separately:
14
+ *
15
+ * - **Is it text?** ({@link classifyFileName} + {@link isProbablyText}) — a known
16
+ * text type is text, a known binary type is binary, and anything else is decided
17
+ * by its first bytes.
18
+ * - **May it be opened?** A `binary` entry is listed as `binary` and the panel says
19
+ * why it cannot be previewed, instead of the row being absent.
20
+ *
8
21
  * @module notebook/story/manuscript
9
22
  */
10
23
  import { basename } from 'node:path';
11
24
  /** Deepest nesting the tree descends into. */
12
- export const MAX_TREE_DEPTH = 8;
25
+ export const MAX_TREE_DEPTH = 12;
13
26
  /** Ceiling on files listed for one root, so a huge tree cannot stall the page. */
14
- export const MAX_TREE_ENTRIES = 2_000;
15
- /** Extensions the browser opens as editable text. */
16
- export const TEXT_EXTENSIONS = new Set(['.md', '.markdown', '.txt', '.json', '.jsonl', '.yml', '.yaml', '.csv']);
27
+ export const MAX_TREE_ENTRIES = 4_000;
28
+ /**
29
+ * How many bytes of an unknown file are sniffed to decide whether it is text.
30
+ *
31
+ * One page is enough to meet the NUL byte of an image, an archive or a compiled
32
+ * binary, and small enough that a listing of a few hundred unknown files is one
33
+ * handful of reads rather than a directory-sized download.
34
+ */
35
+ export const SNIFF_BYTES = 4_096;
36
+ /**
37
+ * Extensions the tree accepts without looking at the bytes.
38
+ *
39
+ * This is an optimization and a default, not the boundary: a name outside it is
40
+ * sniffed rather than refused, so a file type nobody listed still opens when its
41
+ * content is text. The one list that *does* refuse is {@link BINARY_EXTENSIONS}.
42
+ */
43
+ export const TEXT_EXTENSIONS = new Set([
44
+ // Prose and documents.
45
+ '.md', '.markdown', '.mdx', '.txt', '.text', '.rst', '.adoc', '.asciidoc', '.tex', '.org',
46
+ // Data and configuration.
47
+ '.json', '.jsonl', '.jsonc', '.ndjson', '.json5', '.yml', '.yaml', '.toml', '.ini', '.cfg',
48
+ '.conf', '.properties', '.csv', '.tsv', '.xml', '.env', '.diff', '.patch', '.log', '.map',
49
+ // Markup and style.
50
+ '.html', '.htm', '.xhtml', '.css', '.scss', '.sass', '.less', '.styl', '.svg',
51
+ // Source code.
52
+ '.js', '.mjs', '.cjs', '.jsx', '.ts', '.mts', '.cts', '.tsx', '.vue', '.svelte', '.astro',
53
+ '.py', '.rb', '.php', '.go', '.rs', '.java', '.kt', '.kts', '.swift', '.scala', '.groovy',
54
+ '.gradle', '.c', '.h', '.cc', '.cpp', '.cxx', '.hpp', '.hh', '.cs', '.fs', '.fsx', '.lua',
55
+ '.r', '.dart', '.ex', '.exs', '.erl', '.hrl', '.hs', '.clj', '.cljs', '.pl', '.pm', '.zig',
56
+ '.nim', '.jl', '.sh', '.bash', '.zsh', '.fish', '.ps1', '.psm1', '.bat', '.cmd', '.sql',
57
+ '.graphql', '.gql', '.tf', '.hcl', '.nix', '.cmake', '.mk', '.asm', '.s', '.vim', '.sol',
58
+ '.elm', '.v',
59
+ ]);
60
+ /**
61
+ * Extensions the tree never shows, whatever their bytes say.
62
+ *
63
+ * Sniffing would catch most of these anyway — a PNG meets a NUL byte in its
64
+ * header — but a name is free and a read is not: the listing asks the filesystem
65
+ * for a window per unknown file, and images, media and archives are exactly the
66
+ * bulk a workspace of notes accumulates. Everything here is a format whose bytes
67
+ * are not text by construction.
68
+ */
69
+ export const BINARY_EXTENSIONS = new Set([
70
+ // Images.
71
+ '.png', '.jpg', '.jpeg', '.jfif', '.gif', '.webp', '.avif', '.bmp', '.ico', '.icns',
72
+ '.tif', '.tiff', '.psd', '.xcf', '.heic', '.heif', '.raw', '.cr2', '.nef',
73
+ // Documents that are containers rather than text.
74
+ '.pdf', '.doc', '.docx', '.xls', '.xlsx', '.ppt', '.pptx', '.odt', '.ods', '.odp',
75
+ '.epub', '.mobi', '.azw3',
76
+ // Archives and packages.
77
+ '.zip', '.gz', '.tgz', '.bz2', '.xz', '.7z', '.rar', '.tar', '.zst', '.lz4', '.jar', '.war',
78
+ '.apk', '.ipa', '.nupkg', '.vsix', '.deb', '.rpm', '.msi', '.dmg', '.iso', '.squashfs',
79
+ // Compiled and native.
80
+ '.exe', '.dll', '.so', '.dylib', '.bin', '.wasm', '.class', '.o', '.obj', '.a', '.lib',
81
+ '.pdb', '.pyd', '.pyc', '.pyo', '.node', '.elf', '.pak',
82
+ // Audio and video.
83
+ '.mp3', '.wav', '.flac', '.aac', '.ogg', '.oga', '.opus', '.m4a', '.wma', '.mid', '.midi',
84
+ '.mp4', '.m4v', '.mov', '.avi', '.mkv', '.webm', '.wmv', '.flv', '.mpg', '.mpeg', '.3gp',
85
+ // Fonts.
86
+ '.woff', '.woff2', '.ttf', '.otf', '.eot', '.ttc',
87
+ // Databases and binary data sets.
88
+ '.sqlite', '.sqlite3', '.db', '.mdb', '.lmdb', '.parquet', '.avro', '.orc', '.arrow',
89
+ '.npy', '.npz', '.pkl', '.pickle', '.h5', '.hdf5', '.mat', '.sav', '.dta', '.glb', '.fbx',
90
+ ]);
17
91
  /**
18
- * Whether a file is text the editor may open.
92
+ * Classify one file by its name.
19
93
  *
20
- * An allowlist rather than a denylist: a working directory accumulates images,
21
- * archives and OS droppings, and showing all of it would bury the notes. A bare
22
- * name has no extension at all — the panel names files by title (`第一章`) — and
23
- * a title is a text file; only an *unknown* extension is refused.
94
+ * A bare name has no extension at all — the panel names files by title (`第一章`)
95
+ * — and a title is a text file. `.bashrc`-style names are skipped by the listing
96
+ * before this is asked, and a name that starts with a dot has no extension to read
97
+ * here either, so `dot <= 0` is the "no suffix" case.
24
98
  * @param path - project-relative path; only its basename is considered.
25
- * @returns true when the file is editable text.
99
+ * @returns `text`, `binary`, or `unknown` (decide by content).
26
100
  */
27
- export function isTextFile(path) {
101
+ export function classifyFileName(path) {
28
102
  const name = basename(path);
29
103
  const dot = name.lastIndexOf('.');
30
104
  if (dot <= 0)
31
- return true;
32
- return TEXT_EXTENSIONS.has(name.slice(dot).toLowerCase());
105
+ return 'text';
106
+ const extension = name.slice(dot).toLowerCase();
107
+ if (TEXT_EXTENSIONS.has(extension))
108
+ return 'text';
109
+ if (BINARY_EXTENSIONS.has(extension))
110
+ return 'binary';
111
+ return 'unknown';
112
+ }
113
+ /**
114
+ * Whether a name is a text type as far as its extension goes.
115
+ *
116
+ * `unknown` counts as text: the extension said nothing, so only the bytes can, and
117
+ * that read happens against the file rather than the name. The store sniffs an
118
+ * `unknown` file before deciding what to list it as; a binary type is the only
119
+ * thing this predicate turns down.
120
+ * @param path - project-relative path; only its basename is considered.
121
+ * @returns false only for a name known to be binary.
122
+ */
123
+ export function isTextFile(path) {
124
+ return classifyFileName(path) !== 'binary';
125
+ }
126
+ /**
127
+ * Whether a window of bytes looks like text.
128
+ *
129
+ * Two facts decide it, and neither needs the bytes to be UTF-8:
130
+ *
131
+ * - **A NUL byte.** No text format carries one; every binary one this list can
132
+ * meet — images, archives, executables, fonts, media — does, usually within the
133
+ * first few dozen bytes.
134
+ * - **A C0 control other than tab, line feed, vertical tab, form feed and carriage
135
+ * return.** That is the rest of "binary junk" that has no NUL early.
136
+ *
137
+ * Deliberately **not** a UTF-8 validity check: a workspace may hold a Chinese file
138
+ * in GBK or GB18030, which is text a reader expects to open even though it is not
139
+ * UTF-8, and refusing it would hide the author's own notes. The editor decodes
140
+ * whatever the filesystem provider returns.
141
+ * @param head - the first bytes of the file; empty is an empty file, which is text.
142
+ * @returns true when the window passes for text.
143
+ */
144
+ export function isProbablyText(head) {
145
+ for (const byte of head) {
146
+ if (byte === 0)
147
+ return false;
148
+ if (byte < 0x09)
149
+ return false;
150
+ if (byte > 0x0d && byte < 0x20)
151
+ return false;
152
+ if (byte === 0x7f)
153
+ return false;
154
+ }
155
+ return true;
33
156
  }
34
157
  /** Whether an error means "the file is not there". */
35
158
  export function isMissing(error) {
@@ -21,14 +21,24 @@ import { mkdir, rename, rm } from 'node:fs/promises';
21
21
  import { dirname, join } from 'node:path';
22
22
  import { NotebookError, asNotebookError } from '../errors.js';
23
23
  import { assertPathName, assertRelativePath } from './paths.js';
24
- import { countChars, isMissing, isTextFile, MAX_TREE_DEPTH, MAX_TREE_ENTRIES } from './manuscript.js';
24
+ import { countChars, classifyFileName, isMissing, isProbablyText, MAX_TREE_DEPTH, MAX_TREE_ENTRIES, SNIFF_BYTES } from './manuscript.js';
25
25
  /**
26
- * Directories the tree never descends into.
26
+ * Directory names the tree never descends into.
27
27
  *
28
- * `node_modules` is not writing, and a workspace-sized tree that walked it would
29
- * spend seconds per listing and push the author's own files past the entry cap.
28
+ * These are the two trees that are not the author's work and are large enough to
29
+ * bury it: version-control internals and installed packages. Everything else is
30
+ * listed — including dotfiles and dot-folders — because a tree that hides a file
31
+ * the filesystem has is a tree the user cannot trust.
30
32
  */
31
- const SKIP_DIRS = new Set(['node_modules']);
33
+ const SKIP_DIRS = new Set(['.git', '.hg', '.svn', 'node_modules']);
34
+ /**
35
+ * How many unknown-extension files one listing may sniff.
36
+ *
37
+ * A workspace can hold thousands of extensionless blobs, and the tree is drawn on
38
+ * every page load; this keeps that bounded while still covering the normal case —
39
+ * a repository whose files are simply extensions this build has never heard of.
40
+ */
41
+ const MAX_SNIFFS_PER_LISTING = 400;
32
42
  /**
33
43
  * The project id the panel uses for the workspace itself.
34
44
  *
@@ -139,17 +149,35 @@ export class ProjectStore {
139
149
  /**
140
150
  * List every file and directory under the root.
141
151
  *
142
- * Dot entries are tooling rather than notes (`.git`), `node_modules` is not
143
- * writing, and unknown extensions are hidden: a file the tree cannot open is
144
- * noise in a list whose purpose is to be clicked.
152
+ * **Everything is listed.** Dot entries are files and folders like any other
153
+ * (`.gitignore`, `.vscode/`), so they are listed too; only the two tool trees
154
+ * that are not the author's work — version-control internals and installed
155
+ * packages — are skipped, because they would bury the workspace instead of
156
+ * describing it. A file the editor cannot open is still listed, as `binary`:
157
+ * the panel explains on click why it cannot be previewed, which is an answer a
158
+ * reader can act on, while a missing row is indistinguishable from a lost file.
159
+ *
160
+ * Sniffing is for classification, not for filtering, and is bounded per listing:
161
+ * a workspace of thousands of extensionless blobs would otherwise open thousands
162
+ * of files to draw one tree. Past the budget an unknown type is labelled `text`
163
+ * — the panel's read decides for real, and a mislabel costs one explanatory
164
+ * message rather than a hidden file.
145
165
  * @param project - an opened root.
146
- * @returns the entries, sorted by path.
166
+ * @returns the entries, sorted by path, and whether the walk hit its cap.
147
167
  */
148
168
  async listFiles(project) {
149
169
  const out = [];
170
+ // Shared by the whole walk: one budget per listing, not per directory.
171
+ let sniffs = MAX_SNIFFS_PER_LISTING;
172
+ let truncated = false;
150
173
  const walk = async (dir, relDir, depth) => {
151
- if (depth > MAX_TREE_DEPTH || out.length >= MAX_TREE_ENTRIES)
174
+ // Both guards are *partial* listings, not a complete one: nesting past the
175
+ // depth cap or entries past the count cap means files exist that this answer
176
+ // does not contain, and the panel has to be able to say so.
177
+ if (depth > MAX_TREE_DEPTH) {
178
+ truncated = true;
152
179
  return;
180
+ }
153
181
  let entries;
154
182
  try {
155
183
  entries = await this.fs.listDir(dir);
@@ -159,10 +187,10 @@ export class ProjectStore {
159
187
  return;
160
188
  }
161
189
  for (const entry of entries) {
162
- if (out.length >= MAX_TREE_ENTRIES)
190
+ if (out.length >= MAX_TREE_ENTRIES) {
191
+ truncated = true;
163
192
  return;
164
- if (entry.name.startsWith('.'))
165
- continue;
193
+ }
166
194
  if (entry.type === 'directory' && SKIP_DIRS.has(entry.name))
167
195
  continue;
168
196
  // Re-checked per child rather than trusting the walk, so a symlink
@@ -177,31 +205,66 @@ export class ProjectStore {
177
205
  }
178
206
  if (entry.type !== 'file')
179
207
  continue;
180
- if (!isTextFile(entry.name))
181
- continue;
182
208
  const size = entry.size ?? (await this.fs.stat(entry.target).catch(() => undefined))?.size ?? 0;
183
- out.push({ path: rel, name: entry.name, bytes: size, kind: 'text' });
209
+ const classified = classifyFileName(entry.name);
210
+ let kind = 'text';
211
+ if (classified === 'binary') {
212
+ kind = 'binary';
213
+ }
214
+ else if (classified === 'unknown' && sniffs > 0) {
215
+ // An unknown type is decided by its bytes — but listed either way.
216
+ sniffs -= 1;
217
+ kind = (await this.looksText(entry.target)) ? 'text' : 'binary';
218
+ }
219
+ out.push({ path: rel, name: entry.name, bytes: size, kind });
184
220
  }
185
221
  };
186
222
  await walk(project.dir, '', 0);
187
223
  // `numeric` so 第2章 sorts before 第10章 rather than after it, and Chinese
188
- // collation so the folder names group the way a reader expects.
224
+ // collation so the folder names group the way a reader expects. The panel sorts
225
+ // the tree again by name (`state/file-tree`), which is the order the reader
226
+ // sees; this one only keeps the wire format predictable.
189
227
  out.sort((left, right) => left.path.localeCompare(right.path, 'zh-Hans-CN', { numeric: true }));
190
- return out;
228
+ return { files: out, truncated };
229
+ }
230
+ /**
231
+ * Decide by content whether an unknown-extension file is text.
232
+ *
233
+ * The window is a few kilobytes rather than the file: a 1 GiB blob must not be
234
+ * read to draw a row. A read that is refused is treated as text — the refusal
235
+ * says this session may not read the file, not that it is binary, and the open
236
+ * will report the real error at the call site.
237
+ * @param target - the resolved file.
238
+ * @returns true when its first bytes look like text.
239
+ */
240
+ async looksText(target) {
241
+ try {
242
+ const head = await this.fs.readByteRange(target, { offset: 0, length: SNIFF_BYTES });
243
+ return isProbablyText(head);
244
+ }
245
+ catch {
246
+ return true;
247
+ }
191
248
  }
192
249
  /**
193
250
  * Read one text file.
194
251
  * @param project - an opened root.
195
252
  * @param rel - project-relative POSIX path.
196
253
  * @returns the document.
197
- * @throws {NotebookError} `bad-request` when the extension is not editable text,
198
- * `not-found` when the file is absent.
254
+ * @throws {NotebookError} `bad-request` when the file is binary, `not-found` when
255
+ * it is absent.
199
256
  */
200
257
  async readTextFile(project, rel) {
201
258
  const safe = assertRelativePath(rel);
202
- if (!isTextFile(safe))
203
- throw new NotebookError('bad-request', `${safe} is not a text file this editor opens`);
259
+ const kind = classifyFileName(safe);
260
+ if (kind === 'binary')
261
+ throw new NotebookError('bad-request', `${safe} is a binary file this editor does not open`);
204
262
  const target = await this.resolveInside(project, safe);
263
+ // An extension the editor does not know is opened unless its bytes say
264
+ // otherwise — that is the whole point of listing those files.
265
+ if (kind === 'unknown' && !(await this.looksText(target))) {
266
+ throw new NotebookError('bad-request', `${safe} is a binary file this editor does not open`);
267
+ }
205
268
  let content;
206
269
  try {
207
270
  content = await this.fs.readText(target);
@@ -219,12 +282,12 @@ export class ProjectStore {
219
282
  * @param rel - project-relative POSIX path.
220
283
  * @param content - the full new contents.
221
284
  * @returns the stored document.
222
- * @throws {NotebookError} `bad-request` when the extension is not editable text.
285
+ * @throws {NotebookError} `bad-request` when the name is a binary type.
223
286
  */
224
287
  async writeTextFile(project, rel, content) {
225
288
  const safe = assertRelativePath(rel);
226
- if (!isTextFile(safe))
227
- throw new NotebookError('bad-request', `${safe} is not a text file this editor writes`);
289
+ if (classifyFileName(safe) === 'binary')
290
+ throw new NotebookError('bad-request', `${safe} is a binary file this editor does not write`);
228
291
  await this.ensureParent(project, safe);
229
292
  const target = await this.resolveInside(project, safe);
230
293
  try {
package/package.json CHANGED
@@ -1,6 +1,6 @@
1
1
  {
2
2
  "name": "@zmskco/dsh-notebook",
3
- "version": "0.2.0",
3
+ "version": "0.3.0",
4
4
  "description": "笔记本 (Notebook) — a markdown notebook for the DeepSeek Harness workspace: file tree, WYSIWYG editor, autosave and an embedded AI chat.",
5
5
  "icon": "./icon.png",
6
6
  "repository": {
@@ -69,6 +69,7 @@
69
69
  "watch:client": "tsdown --watch",
70
70
  "watch:host": "tsc -p tsconfig.host.json --watch --preserveWatchOutput",
71
71
  "typecheck": "tsc -p tsconfig.host.json --noEmit && tsc -p tsconfig.json",
72
+ "prepare": "pnpm run build",
72
73
  "prepublishOnly": "npm run build && npm test",
73
74
  "pack:check": "npm pack --dry-run"
74
75
  },