@zenera/cli 1.1.22 → 1.1.24
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +72 -2
- package/dist/archive.d.ts +139 -0
- package/dist/archive.js +376 -0
- package/dist/ask.d.ts +51 -0
- package/dist/ask.js +143 -0
- package/dist/batch.d.ts +77 -0
- package/dist/batch.js +338 -0
- package/dist/command.d.ts +6 -0
- package/dist/commands/export.d.ts +3 -0
- package/dist/commands/export.js +255 -0
- package/dist/commands/import.d.ts +3 -0
- package/dist/commands/import.js +173 -0
- package/dist/commands/index.d.ts +5 -1
- package/dist/commands/index.js +13 -1
- package/dist/commands/inspect.d.ts +11 -0
- package/dist/commands/inspect.js +388 -89
- package/dist/commands/list.js +128 -19
- package/dist/commands/memory.js +453 -16
- package/dist/commands/models.js +14 -2
- package/dist/commands/run.js +139 -34
- package/dist/commands/sandbox.js +72 -1
- package/dist/engine.d.ts +19 -0
- package/dist/engine.js +60 -8
- package/dist/liveness.js +6 -3
- package/dist/main.js +5 -2
- package/dist/podman.d.ts +16 -0
- package/dist/podman.js +115 -2
- package/dist/progress.d.ts +33 -0
- package/dist/progress.js +402 -0
- package/dist/request.d.ts +45 -0
- package/dist/request.js +300 -0
- package/dist/sandbox.js +10 -0
- package/dist/scaffold.d.ts +8 -0
- package/dist/scaffold.js +4 -1
- package/dist/session.d.ts +26 -1
- package/dist/session.js +42 -3
- package/dist/term.d.ts +3 -0
- package/dist/term.js +17 -1
- package/dist/trace.d.ts +84 -0
- package/dist/trace.js +686 -0
- package/dist/validate.js +93 -4
- package/package.json +12 -6
- package/templates/editor/.github/copilot-instructions.md +11 -6
- package/templates/editor/.github/prompts/finetune.prompt.md +251 -0
- package/templates/editor/.github/prompts/spec-sync-project.prompt.md +25 -2
- package/templates/editor/.github/skills/zen-cli/SKILL.md +14 -3
- package/templates/editor/.github/skills/zen-cli/references/check.md +13 -5
- package/templates/editor/.github/skills/zen-cli/references/export.md +107 -0
- package/templates/editor/.github/skills/zen-cli/references/inspect.md +262 -21
- package/templates/editor/.github/skills/zen-cli/references/memory.md +123 -15
- package/templates/editor/.github/skills/zen-cli/references/models.md +1 -1
- package/templates/editor/.github/skills/zen-cli/references/projects.md +12 -5
- package/templates/editor/.github/skills/zen-cli/references/rag.md +6 -0
- package/templates/editor/.github/skills/zen-cli/references/run.md +201 -4
- package/templates/editor/.github/skills/zen-finetune/SKILL.md +1000 -0
- package/templates/editor/.github/skills/zen-finetune/scripts/collect.sh +432 -0
- package/templates/editor/.github/skills/zen-finetune/scripts/sample.sh +199 -0
- package/templates/editor/.github/skills/zen-inspect/SKILL.md +396 -0
- package/templates/editor/.github/skills/zen-memory/SKILL.md +74 -28
- package/templates/editor/.github/skills/zen-memory-warmup/SKILL.md +370 -408
- package/templates/editor/.github/skills/zen-sandbox/SKILL.md +19 -0
- package/templates/editor/.github/skills/zen-sandbox-capacity/SKILL.md +334 -0
- package/templates/editor/.github/skills/zen-sandbox-capacity/references/loadtest.md +181 -0
- package/templates/editor/.github/skills/zen-sandbox-capacity/scripts/preflight_sandbox.sh +457 -0
- package/templates/editor/.github/skills/zen-spec-sync/SKILL.md +6 -5
- package/templates/editor/.vscode/agents.schema.json +1 -1
- package/templates/editor/.vscode/settings.json +1 -0
- package/templates/project/SPECIFICATION.md.tmpl +2 -0
- package/templates/project/agents/files-instructions.md +13 -0
- package/templates/project/agents/fork-instructions.md +27 -0
- package/templates/project/agents/memory-instructions.md +80 -7
- package/templates/project/gitignore +3 -0
- package/templates/project/scripts/_setup.sh +49 -7
package/README.md
CHANGED
|
@@ -144,8 +144,11 @@ Then, day to day:
|
|
|
144
144
|
zen run my-project # no prompt given: opens a full-screen terminal app (a TUI)
|
|
145
145
|
zen check my-project # validate the project and every file it names
|
|
146
146
|
zen inspect # open the last run's report.html
|
|
147
|
+
zen inspect graph # the same run as a graph a model can read
|
|
147
148
|
zen list --sessions # every project, its sessions and last run
|
|
149
|
+
zen list --runs # the most recent runs, newest first, across every project
|
|
148
150
|
echo "triage this" | zen run my-project --json | jq
|
|
151
|
+
zen run batch --input cases.json # a file full of questions, 16 at a time
|
|
149
152
|
```
|
|
150
153
|
|
|
151
154
|
Standing inside the project, the name is optional: a bare `zen run`, `zen check`
|
|
@@ -155,6 +158,18 @@ Giving a prompt on the command line skips every question: it starts a fresh
|
|
|
155
158
|
session and uses the current directory as the workspace, with write access.
|
|
156
159
|
`--session`, `--workspace` and `--read-only` override that.
|
|
157
160
|
|
|
161
|
+
`zen run batch` is the same thing for an evaluation set: one project, many
|
|
162
|
+
questions, each in a session, workspace and memory of its own under a batch
|
|
163
|
+
directory. It prints that directory, writes each item's answer the moment it
|
|
164
|
+
lands, and treats a failure as data rather than a stop. Sharing one memory is
|
|
165
|
+
the exception that needs saying out loud - `--memory-read-only` lets every item
|
|
166
|
+
recall from the same graph, because nothing is written and so nothing needs the
|
|
167
|
+
lock. Without it each item gets a copy, and `zen memory merge <batch-dir>/*/memory`
|
|
168
|
+
folds what they learned back in. While it runs, `<batch-dir>/README.md` is a
|
|
169
|
+
dashboard rewritten every second: what each item is doing right now, what the
|
|
170
|
+
finished ones cost - per model, since agents may each use a different one - and
|
|
171
|
+
what is still queued.
|
|
172
|
+
|
|
158
173
|
To change what the system does, update `SPECIFICATION.md` and send
|
|
159
174
|
`/spec-sync-project` again. The next section explains that workflow in detail.
|
|
160
175
|
|
|
@@ -250,6 +265,24 @@ statistics, agent architecture, and memory used during the run.
|
|
|
250
265
|
|
|
251
266
|

|
|
252
267
|
|
|
268
|
+
The agent in your editor can close the same loop without the page. `zen inspect
|
|
269
|
+
graph` writes the whole run as one Mermaid flowchart - a line per node, short
|
|
270
|
+
sequential ids, the tools counted in a header - which is small enough to read
|
|
271
|
+
whole. Having spotted the loop or the branch that failed, it opens the two or
|
|
272
|
+
three nodes that explain it:
|
|
273
|
+
|
|
274
|
+
```sh
|
|
275
|
+
DIR=$(zen run my-project "fix the tests" --json | jq -r .run.dir)
|
|
276
|
+
zen inspect graph --dir "$DIR" # the shape of the run
|
|
277
|
+
zen inspect node n14..n20 --dir "$DIR" # the nodes behind the ids, in full
|
|
278
|
+
zen inspect ask n17 "why python -c here?" --dir "$DIR" # ask the model itself
|
|
279
|
+
```
|
|
280
|
+
|
|
281
|
+
`ask` replays that one call - its recorded system prompt, messages and tool
|
|
282
|
+
schemas - to a model with your question on the end and tool calling off, and
|
|
283
|
+
tells it the run is over and its own instructions are quotable. It is how a
|
|
284
|
+
prompt that reads correctly and behaves wrongly explains itself.
|
|
285
|
+
|
|
253
286
|
Every step is a command, and everything each one reads or writes is a plain
|
|
254
287
|
file: the specification, the findings `zen check` prints with a code, a location
|
|
255
288
|
and a fix, the record `zen inspect` renders of what the run actually did. So the
|
|
@@ -409,7 +442,7 @@ my-project/
|
|
|
409
442
|
sessions/ one workspace, memory and trajectory each
|
|
410
443
|
<id>/
|
|
411
444
|
workspace/ what the agents can read and write
|
|
412
|
-
runs/<id>/ input, output, state, report.html, meta
|
|
445
|
+
runs/<id>/ input, output, state, report.html, graph.mmd, meta
|
|
413
446
|
```
|
|
414
447
|
|
|
415
448
|
Everything above the `sessions/` line is the system; everything below it is what
|
|
@@ -461,6 +494,41 @@ zen sandbox clean # remove containers created by zen
|
|
|
461
494
|
`disk` are machine-wide operations, so use them when you mean to inspect or
|
|
462
495
|
remove resources beyond the current project.
|
|
463
496
|
|
|
497
|
+
## Sharing a project
|
|
498
|
+
|
|
499
|
+
A project is a directory, so sending one somewhere else is a zip - except that
|
|
500
|
+
half of the directory belongs to this machine rather than to the project.
|
|
501
|
+
`zen export` writes the other half.
|
|
502
|
+
|
|
503
|
+
```sh
|
|
504
|
+
zen export # ./<project>-<stamp>.zip, from inside the project
|
|
505
|
+
zen export my-project --out /tmp/a.zip
|
|
506
|
+
zen export --no-vectors # much smaller; needs a restore step on arrival
|
|
507
|
+
|
|
508
|
+
zen import a.zip # unpacks into ./<project> and registers it
|
|
509
|
+
zen import a.zip ~/code/theirs --name theirs
|
|
510
|
+
```
|
|
511
|
+
|
|
512
|
+
It carries `agents.yaml`, the `agents/` tree, the specification and its
|
|
513
|
+
feedback, `assets/`, `memory/`, `sandbox/`, `scripts/` and the editor files.
|
|
514
|
+
It leaves behind `sessions/`, `.tmp/`, `.git/`, `node_modules/`, lock files, any
|
|
515
|
+
`*.zip` at the top of the project - and `.env`. The **values** in `.env` never
|
|
516
|
+
travel, behind no flag; the names do, as a `.env.example` with every value
|
|
517
|
+
blank, because knowing which credentials a project needs is the first thing you
|
|
518
|
+
want on the other end.
|
|
519
|
+
|
|
520
|
+
The vectors travel by default. A rag index without its `lance/` tree looks built
|
|
521
|
+
and cannot search, and memory has no other source at all - a graph without its
|
|
522
|
+
vectors recalls by term overlap until every node is written again. `--no-vectors`
|
|
523
|
+
leaves both out for a much smaller archive, and the import tells you to run
|
|
524
|
+
`zen rag <subject> restore`.
|
|
525
|
+
|
|
526
|
+
`zen import` treats the archive as something a stranger sent: every path is
|
|
527
|
+
checked against escaping the target directory, symbolic-link entries are
|
|
528
|
+
refused, and the size it unpacks to is bounded. **Nothing inside is executed** -
|
|
529
|
+
not `scripts/_setup.sh`, not the Dockerfile, not a line of the `.github/` tree.
|
|
530
|
+
The commands to run next are printed for you to read and type.
|
|
531
|
+
|
|
464
532
|
## Commands
|
|
465
533
|
|
|
466
534
|
| Command | Does |
|
|
@@ -475,6 +543,8 @@ remove resources beyond the current project.
|
|
|
475
543
|
| `inspect` | Opens or rebuilds a run's `report.html`. |
|
|
476
544
|
| `memory` | What the agents remember - size, listing, one node, or a whole page. |
|
|
477
545
|
| `check` | Validates the project and every file it names, and asks the models. |
|
|
546
|
+
| `export` | Writes the project to a shareable zip archive. |
|
|
547
|
+
| `import` | Unpacks one, registers it, and runs nothing in it. |
|
|
478
548
|
| `sandbox` | Checks and prepares the container that command-line tools run in. |
|
|
479
549
|
| `cache` | What work has been kept, and getting rid of it. |
|
|
480
550
|
| `version` | CLI, library and Node versions. |
|
|
@@ -705,7 +775,7 @@ different action:
|
|
|
705
775
|
```
|
|
706
776
|
$ zen models test vertex:gemini-embedding-001
|
|
707
777
|
vertex:gemini-embedding-001 blocked Vertex AI API has not been used in project my-proj …
|
|
708
|
-
vertex:gemini-embedding-001
|
|
778
|
+
fix vertex:gemini-embedding-001 gcloud services enable aiplatform.googleapis.com --project my-proj
|
|
709
779
|
error 1 of 1 did not answer
|
|
710
780
|
find one that does: zen models pick --embedding
|
|
711
781
|
```
|
|
@@ -0,0 +1,139 @@
|
|
|
1
|
+
/** Bumped when the layout below changes in a way an older `zen` cannot read. */
|
|
2
|
+
export declare const ARCHIVE_FORMAT = 1;
|
|
3
|
+
/** The one file at the root of the archive, outside the project directory. */
|
|
4
|
+
export declare const MANIFEST_FILE = "zenera-export.json";
|
|
5
|
+
/** Written in place of `.env`: the names it declared, with no values. */
|
|
6
|
+
export declare const ENV_EXAMPLE = ".env.example";
|
|
7
|
+
/**
|
|
8
|
+
* What the archive says about itself.
|
|
9
|
+
*
|
|
10
|
+
* `root` is the single directory every other entry sits under, and `import`
|
|
11
|
+
* refuses anything outside it — so this is a security boundary as much as a
|
|
12
|
+
* convenience, and it is checked rather than trusted.
|
|
13
|
+
*/
|
|
14
|
+
export interface ArchiveManifest {
|
|
15
|
+
format: number;
|
|
16
|
+
/** the project's name, which is what it is registered as on arrival */
|
|
17
|
+
name: string;
|
|
18
|
+
/** the directory inside the archive holding the project */
|
|
19
|
+
root: string;
|
|
20
|
+
exportedAt: string;
|
|
21
|
+
/** the `zen` that wrote it, for a bug report that starts "it won't open" */
|
|
22
|
+
cli: string;
|
|
23
|
+
/** whether `assets/**\/lance/` travelled */
|
|
24
|
+
vectors: boolean;
|
|
25
|
+
/** whether `memory/` travelled */
|
|
26
|
+
memory: boolean;
|
|
27
|
+
files: number;
|
|
28
|
+
bytes: number;
|
|
29
|
+
}
|
|
30
|
+
/** A file the archive will carry, found by `collect`. */
|
|
31
|
+
export interface ArchiveEntry {
|
|
32
|
+
/** project-relative, always `/`-separated, whatever the platform uses */
|
|
33
|
+
rel: string;
|
|
34
|
+
abs: string;
|
|
35
|
+
bytes: number;
|
|
36
|
+
mode: number;
|
|
37
|
+
}
|
|
38
|
+
export interface CollectOptions {
|
|
39
|
+
/** carry `assets/**\/lance/`; default true */
|
|
40
|
+
vectors?: boolean;
|
|
41
|
+
/** carry `memory/`; default true */
|
|
42
|
+
memory?: boolean;
|
|
43
|
+
}
|
|
44
|
+
export interface Collected {
|
|
45
|
+
files: ArchiveEntry[];
|
|
46
|
+
bytes: number;
|
|
47
|
+
/** how many paths each rule turned away, so the report can say so */
|
|
48
|
+
skipped: Record<string, number>;
|
|
49
|
+
}
|
|
50
|
+
/**
|
|
51
|
+
* Why a path was left out. These are the words the report prints, so they are
|
|
52
|
+
* the ones a person reads when they wonder where something went.
|
|
53
|
+
*/
|
|
54
|
+
export type Skip = 'sessions' | 'scratch' | 'git' | 'modules' | 'env' | 'locks' | 'noise' | 'archives' | 'vectors' | 'memory' | 'links';
|
|
55
|
+
/**
|
|
56
|
+
* The rule, stated once. `rel` is project-relative and `/`-separated, and the
|
|
57
|
+
* answer is the reason to skip it or `undefined` to carry it — for a directory
|
|
58
|
+
* that means the whole subtree.
|
|
59
|
+
*
|
|
60
|
+
* The top-level anchors are deliberate: `sessions/` is run state only when it
|
|
61
|
+
* is *the* sessions directory, and an asset called `sessions` five levels down
|
|
62
|
+
* inside a corpus is a document like any other.
|
|
63
|
+
*/
|
|
64
|
+
export declare function excluded(rel: string, isDir: boolean, opts?: CollectOptions): Skip | undefined;
|
|
65
|
+
/**
|
|
66
|
+
* Walks the project and returns the files that travel, in a deterministic
|
|
67
|
+
* order: two exports of an unchanged tree list the same files in the same
|
|
68
|
+
* sequence, which is what makes a diff of two archives mean anything.
|
|
69
|
+
*
|
|
70
|
+
* Symbolic links are counted and never followed. A link out of the tree is not
|
|
71
|
+
* part of it, and a link back into it would be carried twice — and an archive
|
|
72
|
+
* is extracted by a `zen` that refuses link entries anyway, so writing one
|
|
73
|
+
* would only produce something the other end declines to open.
|
|
74
|
+
*/
|
|
75
|
+
export declare function collect(dir: string, opts?: CollectOptions): Collected;
|
|
76
|
+
/**
|
|
77
|
+
* A `.env` with every value removed.
|
|
78
|
+
*
|
|
79
|
+
* Comments and blank lines survive, because in the file `zen init` writes they
|
|
80
|
+
* are the documentation of what each variable is for, and that is exactly what
|
|
81
|
+
* the person on the other end needs. Anything that is neither a comment nor an
|
|
82
|
+
* assignment is dropped rather than passed through: the only thing it can be
|
|
83
|
+
* is the continuation of a quoted value, and a value is the one thing that
|
|
84
|
+
* must not leave.
|
|
85
|
+
*/
|
|
86
|
+
export declare function envExample(text: string): string;
|
|
87
|
+
/**
|
|
88
|
+
* What a name may be once this tool turns it into a path segment or a registry
|
|
89
|
+
* key. Deliberately permissive — a project is whatever someone called their
|
|
90
|
+
* directory — and closed on the three things that are not names: a traversal,
|
|
91
|
+
* a separator, and a leading dash that some later command would read as a flag.
|
|
92
|
+
*/
|
|
93
|
+
export declare function safeName(value: unknown): string | undefined;
|
|
94
|
+
export interface WriteOptions {
|
|
95
|
+
/** where the archive goes */
|
|
96
|
+
out: string;
|
|
97
|
+
/** the directory inside it that every project file sits under */
|
|
98
|
+
root: string;
|
|
99
|
+
entries: readonly ArchiveEntry[];
|
|
100
|
+
/** files this tool makes up, keyed by their path inside `root` */
|
|
101
|
+
extra?: ReadonlyMap<string, string>;
|
|
102
|
+
manifest: ArchiveManifest;
|
|
103
|
+
onFile?: (rel: string) => void;
|
|
104
|
+
}
|
|
105
|
+
/** Writes the archive and answers with its size on disk. */
|
|
106
|
+
export declare function writeArchive(opts: WriteOptions): Promise<number>;
|
|
107
|
+
/**
|
|
108
|
+
* The absolute path an entry may be written to, or `undefined` to refuse it.
|
|
109
|
+
*
|
|
110
|
+
* Exported because it is the guard, and a guard that is not tested directly is
|
|
111
|
+
* a guard nobody knows the shape of.
|
|
112
|
+
*/
|
|
113
|
+
export declare function safePath(name: string, root: string, into: string): string | undefined;
|
|
114
|
+
/**
|
|
115
|
+
* Reads the manifest, and nothing else. Opening is cheap — it reads the
|
|
116
|
+
* central directory and no file data — so `import` can describe what it is
|
|
117
|
+
* about to do, and decide where to put it, before it writes anything.
|
|
118
|
+
*/
|
|
119
|
+
export declare function readManifest(file: string): Promise<ArchiveManifest>;
|
|
120
|
+
export interface ExtractOptions {
|
|
121
|
+
file: string;
|
|
122
|
+
/** the directory the project's own files land in */
|
|
123
|
+
into: string;
|
|
124
|
+
/** the directory inside the archive they are expected to come from */
|
|
125
|
+
root: string;
|
|
126
|
+
onFile?: (rel: string) => void;
|
|
127
|
+
}
|
|
128
|
+
export interface Extracted {
|
|
129
|
+
files: number;
|
|
130
|
+
bytes: number;
|
|
131
|
+
}
|
|
132
|
+
/**
|
|
133
|
+
* Unpacks the project into `into`, refusing anything the archive claims that
|
|
134
|
+
* this tool would not have written. Nothing is executed, then or afterwards:
|
|
135
|
+
* an archive carries `scripts/` and a `.github/` tree, and the only safe thing
|
|
136
|
+
* to do with a stranger's script is to leave it on disk and say where it is.
|
|
137
|
+
*/
|
|
138
|
+
export declare function extractArchive(opts: ExtractOptions): Promise<Extracted>;
|
|
139
|
+
//# sourceMappingURL=archive.d.ts.map
|
package/dist/archive.js
ADDED
|
@@ -0,0 +1,376 @@
|
|
|
1
|
+
import { createWriteStream, lstatSync, mkdirSync, readdirSync, statSync } from 'node:fs';
|
|
2
|
+
import { mkdir } from 'node:fs/promises';
|
|
3
|
+
import { dirname, join, relative, resolve, sep } from 'node:path';
|
|
4
|
+
import { pipeline } from 'node:stream/promises';
|
|
5
|
+
import * as yauzl from 'yauzl';
|
|
6
|
+
import * as yazl from 'yazl';
|
|
7
|
+
import { ENV_FILE } from "./env.js";
|
|
8
|
+
import { invalidError, usageError } from "./term.js";
|
|
9
|
+
// ---------------------------------------------------------------------------
|
|
10
|
+
// Archives
|
|
11
|
+
//
|
|
12
|
+
// A project is a directory, so sharing one is a question about which of its
|
|
13
|
+
// files are the project and which are the machine it happened to run on. The
|
|
14
|
+
// answer is the same one `.gitignore` gives — sessions, scratch and the `.env`
|
|
15
|
+
// are local; `agents.yaml`, the spec, the assets and what the agents learned
|
|
16
|
+
// are the project — with one difference that matters: the vectors travel. A
|
|
17
|
+
// clone rebuilds a rag index from the documents it carries, but nothing
|
|
18
|
+
// rebuilds memory from its graph, and an archive is not a clone with a setup
|
|
19
|
+
// step waiting for it. It is the whole thing, or it is not worth sending.
|
|
20
|
+
//
|
|
21
|
+
// The format is a zip because every machine already opens one, and because a
|
|
22
|
+
// person who is handed an archive should be able to look inside it without
|
|
23
|
+
// this tool. There is one file at the root that this tool wrote — the manifest
|
|
24
|
+
// — and one directory beside it holding the project verbatim. Nothing else.
|
|
25
|
+
//
|
|
26
|
+
// Everything here is streaming. An assets tree with vectors in it runs to
|
|
27
|
+
// hundreds of megabytes, and a reader that has to hold the archive in memory
|
|
28
|
+
// to open it is a reader that fails on the archives worth sending.
|
|
29
|
+
// ---------------------------------------------------------------------------
|
|
30
|
+
/** Bumped when the layout below changes in a way an older `zen` cannot read. */
|
|
31
|
+
export const ARCHIVE_FORMAT = 1;
|
|
32
|
+
/** The one file at the root of the archive, outside the project directory. */
|
|
33
|
+
export const MANIFEST_FILE = 'zenera-export.json';
|
|
34
|
+
/** Written in place of `.env`: the names it declared, with no values. */
|
|
35
|
+
export const ENV_EXAMPLE = '.env.example';
|
|
36
|
+
/**
|
|
37
|
+
* The rule, stated once. `rel` is project-relative and `/`-separated, and the
|
|
38
|
+
* answer is the reason to skip it or `undefined` to carry it — for a directory
|
|
39
|
+
* that means the whole subtree.
|
|
40
|
+
*
|
|
41
|
+
* The top-level anchors are deliberate: `sessions/` is run state only when it
|
|
42
|
+
* is *the* sessions directory, and an asset called `sessions` five levels down
|
|
43
|
+
* inside a corpus is a document like any other.
|
|
44
|
+
*/
|
|
45
|
+
export function excluded(rel, isDir, opts = {}) {
|
|
46
|
+
const name = rel.slice(rel.lastIndexOf('/') + 1);
|
|
47
|
+
if (isDir) {
|
|
48
|
+
if (rel === 'sessions') {
|
|
49
|
+
return 'sessions';
|
|
50
|
+
}
|
|
51
|
+
if (rel === '.tmp') {
|
|
52
|
+
return 'scratch';
|
|
53
|
+
}
|
|
54
|
+
if (name === '.git') {
|
|
55
|
+
return 'git';
|
|
56
|
+
}
|
|
57
|
+
if (name === 'node_modules') {
|
|
58
|
+
return 'modules';
|
|
59
|
+
}
|
|
60
|
+
if (rel === 'memory' && opts.memory === false) {
|
|
61
|
+
return 'memory';
|
|
62
|
+
}
|
|
63
|
+
if (name === 'lance' && rel.startsWith('assets/') && opts.vectors === false) {
|
|
64
|
+
return 'vectors';
|
|
65
|
+
}
|
|
66
|
+
return undefined;
|
|
67
|
+
}
|
|
68
|
+
// The credentials never travel, in any form and behind no flag. What the
|
|
69
|
+
// project needs is described by `.env.example`, which `export` synthesizes
|
|
70
|
+
// from the names in this file.
|
|
71
|
+
if (rel === ENV_FILE) {
|
|
72
|
+
return 'env';
|
|
73
|
+
}
|
|
74
|
+
if (name === '.lock') {
|
|
75
|
+
return 'locks';
|
|
76
|
+
}
|
|
77
|
+
if (name === '.DS_Store') {
|
|
78
|
+
return 'noise';
|
|
79
|
+
}
|
|
80
|
+
// An archive at the top of the project is one of these, and the default
|
|
81
|
+
// destination puts it there: `zen export` run twice from inside a project
|
|
82
|
+
// would otherwise pack the first archive into the second. Project material
|
|
83
|
+
// that happens to be a zip lives under `assets/`, where this does not look.
|
|
84
|
+
if (!rel.includes('/') && name.toLowerCase().endsWith('.zip')) {
|
|
85
|
+
return 'archives';
|
|
86
|
+
}
|
|
87
|
+
return undefined;
|
|
88
|
+
}
|
|
89
|
+
/**
|
|
90
|
+
* Walks the project and returns the files that travel, in a deterministic
|
|
91
|
+
* order: two exports of an unchanged tree list the same files in the same
|
|
92
|
+
* sequence, which is what makes a diff of two archives mean anything.
|
|
93
|
+
*
|
|
94
|
+
* Symbolic links are counted and never followed. A link out of the tree is not
|
|
95
|
+
* part of it, and a link back into it would be carried twice — and an archive
|
|
96
|
+
* is extracted by a `zen` that refuses link entries anyway, so writing one
|
|
97
|
+
* would only produce something the other end declines to open.
|
|
98
|
+
*/
|
|
99
|
+
export function collect(dir, opts = {}) {
|
|
100
|
+
const root = resolve(dir);
|
|
101
|
+
const files = [];
|
|
102
|
+
const skipped = {};
|
|
103
|
+
const drop = (reason) => {
|
|
104
|
+
skipped[reason] = (skipped[reason] ?? 0) + 1;
|
|
105
|
+
};
|
|
106
|
+
const walk = (at) => {
|
|
107
|
+
const entries = readdirSync(at, { withFileTypes: true });
|
|
108
|
+
// Codepoint order, not locale order: the answer must not depend on
|
|
109
|
+
// which machine ran the export.
|
|
110
|
+
entries.sort((a, b) => (a.name < b.name ? -1 : a.name > b.name ? 1 : 0));
|
|
111
|
+
for (const entry of entries) {
|
|
112
|
+
const abs = join(at, entry.name);
|
|
113
|
+
const rel = relative(root, abs).split(sep).join('/');
|
|
114
|
+
if (entry.isSymbolicLink()) {
|
|
115
|
+
drop('links');
|
|
116
|
+
continue;
|
|
117
|
+
}
|
|
118
|
+
const reason = excluded(rel, entry.isDirectory(), opts);
|
|
119
|
+
if (reason) {
|
|
120
|
+
drop(reason);
|
|
121
|
+
continue;
|
|
122
|
+
}
|
|
123
|
+
if (entry.isDirectory()) {
|
|
124
|
+
walk(abs);
|
|
125
|
+
continue;
|
|
126
|
+
}
|
|
127
|
+
if (!entry.isFile()) {
|
|
128
|
+
// A socket or a fifo left behind by something. Not a file.
|
|
129
|
+
drop('noise');
|
|
130
|
+
continue;
|
|
131
|
+
}
|
|
132
|
+
let stat;
|
|
133
|
+
try {
|
|
134
|
+
stat = lstatSync(abs);
|
|
135
|
+
}
|
|
136
|
+
catch {
|
|
137
|
+
// Gone between the listing and the stat. Nothing to carry.
|
|
138
|
+
continue;
|
|
139
|
+
}
|
|
140
|
+
files.push({ rel, abs, bytes: stat.size, mode: stat.mode & 0o777 });
|
|
141
|
+
}
|
|
142
|
+
};
|
|
143
|
+
walk(root);
|
|
144
|
+
return { files, bytes: files.reduce((n, f) => n + f.bytes, 0), skipped };
|
|
145
|
+
}
|
|
146
|
+
/**
|
|
147
|
+
* A `.env` with every value removed.
|
|
148
|
+
*
|
|
149
|
+
* Comments and blank lines survive, because in the file `zen init` writes they
|
|
150
|
+
* are the documentation of what each variable is for, and that is exactly what
|
|
151
|
+
* the person on the other end needs. Anything that is neither a comment nor an
|
|
152
|
+
* assignment is dropped rather than passed through: the only thing it can be
|
|
153
|
+
* is the continuation of a quoted value, and a value is the one thing that
|
|
154
|
+
* must not leave.
|
|
155
|
+
*/
|
|
156
|
+
export function envExample(text) {
|
|
157
|
+
const lines = [];
|
|
158
|
+
for (const line of text.split(/\r?\n/)) {
|
|
159
|
+
const trimmed = line.trimStart();
|
|
160
|
+
if (trimmed === '' || trimmed.startsWith('#')) {
|
|
161
|
+
lines.push(line.trimEnd());
|
|
162
|
+
continue;
|
|
163
|
+
}
|
|
164
|
+
const assigned = /^(\s*(?:export\s+)?[A-Za-z_][A-Za-z0-9_]*)\s*=/.exec(line);
|
|
165
|
+
if (assigned) {
|
|
166
|
+
lines.push(`${assigned[1]}=`);
|
|
167
|
+
}
|
|
168
|
+
}
|
|
169
|
+
return `${lines.join('\n').trimEnd()}\n`;
|
|
170
|
+
}
|
|
171
|
+
/**
|
|
172
|
+
* What a name may be once this tool turns it into a path segment or a registry
|
|
173
|
+
* key. Deliberately permissive — a project is whatever someone called their
|
|
174
|
+
* directory — and closed on the three things that are not names: a traversal,
|
|
175
|
+
* a separator, and a leading dash that some later command would read as a flag.
|
|
176
|
+
*/
|
|
177
|
+
export function safeName(value) {
|
|
178
|
+
if (typeof value !== 'string' || value.length === 0 || value.length > 128) {
|
|
179
|
+
return undefined;
|
|
180
|
+
}
|
|
181
|
+
if (value === '.' || value === '..' || value.startsWith('-')) {
|
|
182
|
+
return undefined;
|
|
183
|
+
}
|
|
184
|
+
return /[/\\\0]/.test(value) ? undefined : value;
|
|
185
|
+
}
|
|
186
|
+
/** Writes the archive and answers with its size on disk. */
|
|
187
|
+
export async function writeArchive(opts) {
|
|
188
|
+
const zip = new yazl.ZipFile();
|
|
189
|
+
const out = resolve(opts.out);
|
|
190
|
+
mkdirSync(dirname(out), { recursive: true });
|
|
191
|
+
// Piped before anything is added: yazl reads eagerly, and an unread
|
|
192
|
+
// outputStream is the whole archive held in memory.
|
|
193
|
+
const sink = createWriteStream(out);
|
|
194
|
+
const finished = pipeline(zip.outputStream, sink);
|
|
195
|
+
zip.addBuffer(Buffer.from(`${JSON.stringify(opts.manifest, null, 4)}\n`, 'utf8'), MANIFEST_FILE, {
|
|
196
|
+
mode: 0o644,
|
|
197
|
+
});
|
|
198
|
+
for (const entry of opts.entries) {
|
|
199
|
+
zip.addFile(entry.abs, `${opts.root}/${entry.rel}`, { mode: entry.mode });
|
|
200
|
+
opts.onFile?.(entry.rel);
|
|
201
|
+
}
|
|
202
|
+
for (const [rel, text] of opts.extra ?? []) {
|
|
203
|
+
zip.addBuffer(Buffer.from(text, 'utf8'), `${opts.root}/${rel}`, { mode: 0o644 });
|
|
204
|
+
}
|
|
205
|
+
zip.end();
|
|
206
|
+
await finished;
|
|
207
|
+
return statSync(out).size;
|
|
208
|
+
}
|
|
209
|
+
// ---------------------------------------------------------------------------
|
|
210
|
+
// Reading
|
|
211
|
+
//
|
|
212
|
+
// Everything below treats the archive as something a stranger sent, because
|
|
213
|
+
// that is the only reason it exists. A zip entry carries its own path, its own
|
|
214
|
+
// size and its own unix mode, and all three are claims: the path can walk out
|
|
215
|
+
// of the directory it was extracted into, the size can be a lie that unpacks
|
|
216
|
+
// to more than the disk holds, and the mode can say "this is a symbolic link",
|
|
217
|
+
// which is how an unpacker is talked into writing outside the tree it checked.
|
|
218
|
+
// ---------------------------------------------------------------------------
|
|
219
|
+
/** No project has this many files, and no archive this tool wrote does. */
|
|
220
|
+
const MAX_ENTRIES = 200_000;
|
|
221
|
+
/** Nor this many bytes once unpacked. Both bounds exist to stop a zip bomb. */
|
|
222
|
+
const MAX_BYTES = 8 * 1024 ** 3;
|
|
223
|
+
/** A manifest is a small json file; anything larger is not one. */
|
|
224
|
+
const MAX_MANIFEST = 64 * 1024;
|
|
225
|
+
/** The file-type bits of a unix mode, and the value that means "symlink". */
|
|
226
|
+
const S_IFMT = 0o170000;
|
|
227
|
+
const S_IFLNK = 0o120000;
|
|
228
|
+
/**
|
|
229
|
+
* The absolute path an entry may be written to, or `undefined` to refuse it.
|
|
230
|
+
*
|
|
231
|
+
* Exported because it is the guard, and a guard that is not tested directly is
|
|
232
|
+
* a guard nobody knows the shape of.
|
|
233
|
+
*/
|
|
234
|
+
export function safePath(name, root, into) {
|
|
235
|
+
if (name.length === 0 || name.includes('\\') || name.includes('\0')) {
|
|
236
|
+
return undefined;
|
|
237
|
+
}
|
|
238
|
+
if (name.startsWith('/') || /^[A-Za-z]:/.test(name)) {
|
|
239
|
+
return undefined;
|
|
240
|
+
}
|
|
241
|
+
const parts = name.split('/');
|
|
242
|
+
if (parts.some((part) => part === '' || part === '.' || part === '..')) {
|
|
243
|
+
return undefined;
|
|
244
|
+
}
|
|
245
|
+
if (parts.length < 2 || parts[0] !== root) {
|
|
246
|
+
return undefined;
|
|
247
|
+
}
|
|
248
|
+
// Every segment has been checked, so this cannot fail — which is the point
|
|
249
|
+
// of asking. A guard whose last step is redundant is one that still holds
|
|
250
|
+
// when an earlier step is changed by someone who missed the reason for it.
|
|
251
|
+
const base = resolve(into);
|
|
252
|
+
const at = resolve(base, ...parts.slice(1));
|
|
253
|
+
return at.startsWith(base + sep) ? at : undefined;
|
|
254
|
+
}
|
|
255
|
+
/**
|
|
256
|
+
* Reads the manifest, and nothing else. Opening is cheap — it reads the
|
|
257
|
+
* central directory and no file data — so `import` can describe what it is
|
|
258
|
+
* about to do, and decide where to put it, before it writes anything.
|
|
259
|
+
*/
|
|
260
|
+
export async function readManifest(file) {
|
|
261
|
+
const zip = await yauzl.openPromise(file, { lazyEntries: true }).catch((err) => {
|
|
262
|
+
throw usageError(`${file} is not a readable zip archive`, err instanceof Error ? err.message : undefined);
|
|
263
|
+
});
|
|
264
|
+
let found;
|
|
265
|
+
for await (const entry of zip.eachEntry()) {
|
|
266
|
+
if (entry.fileName !== MANIFEST_FILE) {
|
|
267
|
+
continue;
|
|
268
|
+
}
|
|
269
|
+
if (entry.uncompressedSize > MAX_MANIFEST) {
|
|
270
|
+
zip.close();
|
|
271
|
+
throw invalidError(`${MANIFEST_FILE} is implausibly large`);
|
|
272
|
+
}
|
|
273
|
+
found = await read(await zip.openReadStreamPromise(entry));
|
|
274
|
+
zip.close();
|
|
275
|
+
break;
|
|
276
|
+
}
|
|
277
|
+
if (found === undefined) {
|
|
278
|
+
throw usageError(`${file} has no ${MANIFEST_FILE}`, 'only an archive written by `zen export` can be imported');
|
|
279
|
+
}
|
|
280
|
+
return parseManifest(found);
|
|
281
|
+
}
|
|
282
|
+
/** Checked rather than trusted: every field here arrives from the archive. */
|
|
283
|
+
function parseManifest(text) {
|
|
284
|
+
let raw;
|
|
285
|
+
try {
|
|
286
|
+
raw = JSON.parse(text);
|
|
287
|
+
}
|
|
288
|
+
catch {
|
|
289
|
+
throw invalidError(`${MANIFEST_FILE} is not valid json`);
|
|
290
|
+
}
|
|
291
|
+
const format = typeof raw.format === 'number' ? raw.format : 0;
|
|
292
|
+
if (format > ARCHIVE_FORMAT) {
|
|
293
|
+
throw invalidError(`this archive is format ${format}, and this zen reads ${ARCHIVE_FORMAT}`, 'upgrade: npm i -g @zenera/cli');
|
|
294
|
+
}
|
|
295
|
+
const name = safeName(raw.name);
|
|
296
|
+
const root = safeName(raw.root);
|
|
297
|
+
if (!name || !root) {
|
|
298
|
+
throw invalidError(`${MANIFEST_FILE} names the project in a way zen will not write`);
|
|
299
|
+
}
|
|
300
|
+
return {
|
|
301
|
+
format,
|
|
302
|
+
name,
|
|
303
|
+
root,
|
|
304
|
+
exportedAt: typeof raw.exportedAt === 'string' ? raw.exportedAt : '',
|
|
305
|
+
cli: typeof raw.cli === 'string' ? raw.cli : 'unknown',
|
|
306
|
+
vectors: raw.vectors !== false,
|
|
307
|
+
memory: raw.memory !== false,
|
|
308
|
+
files: typeof raw.files === 'number' ? raw.files : 0,
|
|
309
|
+
bytes: typeof raw.bytes === 'number' ? raw.bytes : 0,
|
|
310
|
+
};
|
|
311
|
+
}
|
|
312
|
+
/**
|
|
313
|
+
* Unpacks the project into `into`, refusing anything the archive claims that
|
|
314
|
+
* this tool would not have written. Nothing is executed, then or afterwards:
|
|
315
|
+
* an archive carries `scripts/` and a `.github/` tree, and the only safe thing
|
|
316
|
+
* to do with a stranger's script is to leave it on disk and say where it is.
|
|
317
|
+
*/
|
|
318
|
+
export async function extractArchive(opts) {
|
|
319
|
+
const zip = await yauzl.openPromise(opts.file, {
|
|
320
|
+
lazyEntries: true,
|
|
321
|
+
validateEntrySizes: true,
|
|
322
|
+
});
|
|
323
|
+
let files = 0;
|
|
324
|
+
let bytes = 0;
|
|
325
|
+
let drained = false;
|
|
326
|
+
try {
|
|
327
|
+
for await (const entry of zip.eachEntry()) {
|
|
328
|
+
// Directories arrive as their own entries; the tree is made from
|
|
329
|
+
// the file paths instead, so an empty one simply does not travel.
|
|
330
|
+
if (entry.fileName === MANIFEST_FILE || entry.fileName.endsWith('/')) {
|
|
331
|
+
continue;
|
|
332
|
+
}
|
|
333
|
+
if (entry.isEncrypted()) {
|
|
334
|
+
throw invalidError(`${entry.fileName} is encrypted`, 'zen does not write those');
|
|
335
|
+
}
|
|
336
|
+
if (++files > MAX_ENTRIES) {
|
|
337
|
+
throw invalidError(`this archive holds more than ${MAX_ENTRIES} files`);
|
|
338
|
+
}
|
|
339
|
+
bytes += entry.uncompressedSize;
|
|
340
|
+
if (bytes > MAX_BYTES) {
|
|
341
|
+
throw invalidError('this archive unpacks to more than 8 GiB');
|
|
342
|
+
}
|
|
343
|
+
if (((entry.externalFileAttributes >>> 16) & S_IFMT) === S_IFLNK) {
|
|
344
|
+
throw invalidError(`${entry.fileName} is a symbolic link`, 'a link can point out of the directory being written to');
|
|
345
|
+
}
|
|
346
|
+
const to = safePath(entry.fileName, opts.root, opts.into);
|
|
347
|
+
if (!to) {
|
|
348
|
+
throw invalidError(`${entry.fileName} does not belong under ${opts.root}/`, 'this archive was not written by `zen export`');
|
|
349
|
+
}
|
|
350
|
+
await mkdir(dirname(to), { recursive: true });
|
|
351
|
+
const stream = await zip.openReadStreamPromise(entry);
|
|
352
|
+
// The mode is the archive's claim, so it is replaced rather than
|
|
353
|
+
// honoured. A shell script has to be runnable to be useful; there
|
|
354
|
+
// is no other reason for anything here to carry a bit.
|
|
355
|
+
const mode = entry.fileName.endsWith('.sh') ? 0o755 : 0o644;
|
|
356
|
+
await pipeline(stream, createWriteStream(to, { mode }));
|
|
357
|
+
opts.onFile?.(entry.fileName.slice(opts.root.length + 1));
|
|
358
|
+
}
|
|
359
|
+
drained = true;
|
|
360
|
+
}
|
|
361
|
+
finally {
|
|
362
|
+
// Reaching the end closes it; leaving early does not.
|
|
363
|
+
if (!drained) {
|
|
364
|
+
zip.close();
|
|
365
|
+
}
|
|
366
|
+
}
|
|
367
|
+
return { files, bytes };
|
|
368
|
+
}
|
|
369
|
+
async function read(stream) {
|
|
370
|
+
const chunks = [];
|
|
371
|
+
for await (const chunk of stream) {
|
|
372
|
+
chunks.push(chunk);
|
|
373
|
+
}
|
|
374
|
+
return Buffer.concat(chunks).toString('utf8');
|
|
375
|
+
}
|
|
376
|
+
//# sourceMappingURL=archive.js.map
|