larkit 0.0.0-stage → 0.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/AGENT_SKILL.md +12 -0
- package/README.md +765 -2
- package/bin/larkit-mcp.js +8 -0
- package/bin/larkit.js +532 -0
- package/larkit.config.example.json +12 -0
- package/package.json +65 -4
- package/scripts/.tmp-npm-webauth.mjs +49 -0
- package/scripts/build-index.mjs +66 -0
- package/scripts/check-syntax.mjs +28 -0
- package/scripts/login-worker.mjs +50 -0
- package/scripts/publish-pack.mjs +277 -0
- package/scripts/verify-bootstrap.mjs +238 -0
- package/scripts/verify-console.mjs +60 -0
- package/scripts/verify-index.mjs +59 -0
- package/scripts/verify-mcp.mjs +215 -0
- package/skills/larkit/SKILL.md +120 -0
- package/skills/larkit/references/console.md +86 -0
- package/skills/larkit/references/playbooks.md +177 -0
- package/skills/larkit/references/tools.md +115 -0
- package/skills/larkit/references/truth-model.md +117 -0
- package/src/auth/login.js +171 -0
- package/src/auth/service.js +213 -0
- package/src/auth/store.js +213 -0
- package/src/bootstrap/channel.js +201 -0
- package/src/bootstrap/cli-install.js +184 -0
- package/src/bootstrap/download.js +182 -0
- package/src/bootstrap/init.js +672 -0
- package/src/bootstrap/pack-cli.js +97 -0
- package/src/bootstrap/pack.js +541 -0
- package/src/bootstrap/report.js +62 -0
- package/src/bootstrap/skills.js +223 -0
- package/src/bootstrap/state.js +66 -0
- package/src/bootstrap/tar.js +417 -0
- package/src/console/browser.js +338 -0
- package/src/console/client.js +827 -0
- package/src/console/dispatch.js +101 -0
- package/src/docs/client.js +284 -0
- package/src/indexer/anchors.js +366 -0
- package/src/indexer/db.js +449 -0
- package/src/indexer/extract-assets.js +285 -0
- package/src/indexer/extract-deb.js +56 -0
- package/src/indexer/format-pool.js +64 -0
- package/src/indexer/format-worker.js +132 -0
- package/src/indexer/format.js +239 -0
- package/src/indexer/protocol.js +123 -0
- package/src/mcp/server.js +433 -0
- package/src/probe/client.js +225 -0
- package/src/probe/openapi.js +194 -0
- package/src/protocol/catalog.js +146 -0
- package/src/protocol/data/README.md +15 -0
- package/src/protocol/data/bundle.json +1 -0
- package/src/protocol/data/commands.json +12896 -0
- package/src/protocol/data/constants.json +8 -0
- package/src/protocol/data/reference/PROTOCOL.md +66 -0
- package/src/protocol/data/reference/endpoints.json +46 -0
- package/src/protocol/data/schema.json.gz +0 -0
- package/src/protocol/descriptor.js +358 -0
- package/src/protocol/schema.js +235 -0
- package/src/protocol/transport.js +387 -0
- package/src/shared/browser.js +70 -0
- package/src/shared/config.js +90 -0
- package/src/shared/paths.js +151 -0
- package/src/shared/quiet.js +12 -0
- package/src/shared/util.js +124 -0
- package/src/truth/fts.js +24 -0
- package/src/truth/search.js +554 -0
- package/src/truth/verified.js +107 -0
- package/verified_truths/README.md +28 -0
- package/verified_truths/card-json-2.0-official.md +55 -0
- package/verified_truths/client-ws-gateway-boundaries.md +33 -0
- package/verified_truths/feishu-developer-console-apis.md +104 -0
- package/workers/truth-api/src/index.js +180 -0
- package/workers/truth-api/wrangler.toml +31 -0
|
@@ -0,0 +1,97 @@
|
|
|
1
|
+
import fs from "node:fs";
|
|
2
|
+
import path from "node:path";
|
|
3
|
+
import {
|
|
4
|
+
DATA_DIR,
|
|
5
|
+
DOWNLOAD_DIR,
|
|
6
|
+
FORMATTED_DIR,
|
|
7
|
+
LARKIT_HOME,
|
|
8
|
+
PKG_ROOT,
|
|
9
|
+
TRUTH_DB,
|
|
10
|
+
VERIFIED_DIR,
|
|
11
|
+
} from "../shared/paths.js";
|
|
12
|
+
import { readJson } from "../shared/util.js";
|
|
13
|
+
import { buildTruthPack, packageVersion, verifyPack } from "./pack.js";
|
|
14
|
+
|
|
15
|
+
function defaultOutDir() {
|
|
16
|
+
return path.join(PKG_ROOT, "dist", "truth-pack");
|
|
17
|
+
}
|
|
18
|
+
|
|
19
|
+
export async function runPackCommand(action, flags = {}, reporter) {
|
|
20
|
+
const version = packageVersion();
|
|
21
|
+
|
|
22
|
+
if (action === "path") {
|
|
23
|
+
const value = {
|
|
24
|
+
larkitVersion: version,
|
|
25
|
+
home: LARKIT_HOME,
|
|
26
|
+
dataDir: DATA_DIR,
|
|
27
|
+
truthDb: TRUTH_DB,
|
|
28
|
+
verifiedDir: VERIFIED_DIR,
|
|
29
|
+
formattedDir: FORMATTED_DIR,
|
|
30
|
+
downloads: DOWNLOAD_DIR,
|
|
31
|
+
packageRoot: PKG_ROOT,
|
|
32
|
+
};
|
|
33
|
+
reporter.result(value);
|
|
34
|
+
if (!reporter.json) {
|
|
35
|
+
for (const [key, entry] of Object.entries(value)) {
|
|
36
|
+
reporter.detail(key, entry);
|
|
37
|
+
}
|
|
38
|
+
}
|
|
39
|
+
return value;
|
|
40
|
+
}
|
|
41
|
+
|
|
42
|
+
if (action === "verify") {
|
|
43
|
+
const source =
|
|
44
|
+
flags.manifest || process.env.LARKIT_PACK_MANIFEST || flags.from || null;
|
|
45
|
+
if (!source) {
|
|
46
|
+
throw new Error("usage: larkit pack verify <manifest.json|pack-dir|.tar.gz|https://...>");
|
|
47
|
+
}
|
|
48
|
+
const report = await verifyPack(source);
|
|
49
|
+
reporter.result(report);
|
|
50
|
+
if (!reporter.json) {
|
|
51
|
+
for (const tier of report.tiers) {
|
|
52
|
+
reporter.line(
|
|
53
|
+
`${tier.ok ? "PASS" : "FAIL"} ${tier.tier} ${tier.file || tier.url}${tier.remote ? " (remote, not downloaded)" : ""}`,
|
|
54
|
+
);
|
|
55
|
+
}
|
|
56
|
+
}
|
|
57
|
+
if (!report.ok) throw new Error("pack verification failed");
|
|
58
|
+
return report;
|
|
59
|
+
}
|
|
60
|
+
|
|
61
|
+
if (action === "build") {
|
|
62
|
+
const outDir = flags.out ? path.resolve(flags.out) : defaultOutDir();
|
|
63
|
+
const built = await buildTruthPack({
|
|
64
|
+
outDir,
|
|
65
|
+
baseUrl: flags["base-url"] || "",
|
|
66
|
+
tag: flags.tag || "",
|
|
67
|
+
version,
|
|
68
|
+
onProgress: (bytes) => reporter.progress?.("pack", bytes, 0),
|
|
69
|
+
});
|
|
70
|
+
reporter.result(built);
|
|
71
|
+
if (!reporter.json) {
|
|
72
|
+
reporter.line("");
|
|
73
|
+
reporter.ok(`manifest ${built.manifestFile}`);
|
|
74
|
+
for (const [tier, artifact] of Object.entries(built.manifest.tiers)) {
|
|
75
|
+
reporter.detail(
|
|
76
|
+
tier,
|
|
77
|
+
`${artifact.file} ${(artifact.bytes / 1024 / 1024).toFixed(1)} MiB ${artifact.sha256.slice(0, 16)}…`,
|
|
78
|
+
);
|
|
79
|
+
}
|
|
80
|
+
}
|
|
81
|
+
return built;
|
|
82
|
+
}
|
|
83
|
+
|
|
84
|
+
if (action === "stats") {
|
|
85
|
+
const manifest = readJson(
|
|
86
|
+
flags.manifest ||
|
|
87
|
+
process.env.LARKIT_PACK_MANIFEST ||
|
|
88
|
+
path.join(defaultOutDir(), "manifest.json"),
|
|
89
|
+
null,
|
|
90
|
+
);
|
|
91
|
+
if (!manifest) throw new Error("no pack manifest found; run `larkit pack build`");
|
|
92
|
+
reporter.result(manifest.stats || {});
|
|
93
|
+
return manifest.stats || {};
|
|
94
|
+
}
|
|
95
|
+
|
|
96
|
+
throw new Error("usage: larkit pack <build|verify|path|stats>");
|
|
97
|
+
}
|
|
@@ -0,0 +1,541 @@
|
|
|
1
|
+
import fs from "node:fs";
|
|
2
|
+
import path from "node:path";
|
|
3
|
+
import { fileURLToPath } from "node:url";
|
|
4
|
+
import {
|
|
5
|
+
DATA_DIR,
|
|
6
|
+
DOWNLOAD_DIR,
|
|
7
|
+
FORMATTED_DIR,
|
|
8
|
+
PKG_ROOT,
|
|
9
|
+
TRUTH_DB,
|
|
10
|
+
TRUTH_META,
|
|
11
|
+
VERIFIED_DIR,
|
|
12
|
+
displayPath,
|
|
13
|
+
} from "../shared/paths.js";
|
|
14
|
+
import {
|
|
15
|
+
ensureDir,
|
|
16
|
+
fileSizeMiB,
|
|
17
|
+
log,
|
|
18
|
+
readJson,
|
|
19
|
+
walkFiles,
|
|
20
|
+
writeJson,
|
|
21
|
+
} from "../shared/util.js";
|
|
22
|
+
import { createTarGz, extractTarGz } from "./tar.js";
|
|
23
|
+
import {
|
|
24
|
+
downloadFile,
|
|
25
|
+
fetchJson,
|
|
26
|
+
isFileUrl,
|
|
27
|
+
isHttpUrl,
|
|
28
|
+
sha256File,
|
|
29
|
+
sha256FileSync,
|
|
30
|
+
} from "./download.js";
|
|
31
|
+
|
|
32
|
+
export const PACK_KIND = "larkit-truth-pack";
|
|
33
|
+
export const PACK_SCHEMA_VERSION = 1;
|
|
34
|
+
export const TIER_NAME = "full";
|
|
35
|
+
/**
|
|
36
|
+
* The official truth channel: `larkit init --channel` (or `--pack=<url>`)
|
|
37
|
+
* pulls the newest published truth pack from here, independently of npm
|
|
38
|
+
* releases. Truth packs are published by maintainers (see README §14.1).
|
|
39
|
+
*/
|
|
40
|
+
export const DEFAULT_TRUTH_CHANNEL =
|
|
41
|
+
"https://larkit.tapeless.eu.org/v1/truth/latest";
|
|
42
|
+
const SQLITE_MAGIC = Buffer.from("SQLite format 3\0", "utf8");
|
|
43
|
+
|
|
44
|
+
export function isChannelSource(source) {
|
|
45
|
+
return typeof source === "string" && isHttpUrl(source);
|
|
46
|
+
}
|
|
47
|
+
|
|
48
|
+
export function packageVersion() {
|
|
49
|
+
return readJson(path.join(PKG_ROOT, "package.json"), {}).version || "0.0.0";
|
|
50
|
+
}
|
|
51
|
+
|
|
52
|
+
export function truthMeta() {
|
|
53
|
+
return readJson(TRUTH_META, {});
|
|
54
|
+
}
|
|
55
|
+
|
|
56
|
+
export function tierFileName(tag) {
|
|
57
|
+
return `larkit-truth-${TIER_NAME}-${tag}.tar.gz`;
|
|
58
|
+
}
|
|
59
|
+
|
|
60
|
+
/**
|
|
61
|
+
* Truth packs are released independently of npm releases, so they carry their
|
|
62
|
+
* own immutable tag (e.g. "2026.11.15"). Archive names are permanent URLs, so
|
|
63
|
+
* a tag must never be reused for different bytes.
|
|
64
|
+
*/
|
|
65
|
+
function normalizeTag(tag) {
|
|
66
|
+
const value = String(tag || "").trim();
|
|
67
|
+
if (!value) return "";
|
|
68
|
+
if (!/^[A-Za-z0-9][A-Za-z0-9._-]{0,63}$/.test(value)) {
|
|
69
|
+
throw new Error(
|
|
70
|
+
`invalid pack tag "${value}": use letters, digits, dot, dash or underscore (max 64 chars)`,
|
|
71
|
+
);
|
|
72
|
+
}
|
|
73
|
+
return value;
|
|
74
|
+
}
|
|
75
|
+
|
|
76
|
+
function markdownFiles(dir) {
|
|
77
|
+
if (!fs.existsSync(dir)) return [];
|
|
78
|
+
return fs
|
|
79
|
+
.readdirSync(dir)
|
|
80
|
+
.filter((name) => name.endsWith(".md"))
|
|
81
|
+
.sort()
|
|
82
|
+
.map((name) => path.join(dir, name));
|
|
83
|
+
}
|
|
84
|
+
|
|
85
|
+
function assertSqlite(file) {
|
|
86
|
+
const fd = fs.openSync(file, "r");
|
|
87
|
+
try {
|
|
88
|
+
const header = Buffer.alloc(16);
|
|
89
|
+
fs.readSync(fd, header, 0, 16, 0);
|
|
90
|
+
if (!header.equals(SQLITE_MAGIC)) {
|
|
91
|
+
throw new Error(`${displayPath(file)} is not a SQLite database`);
|
|
92
|
+
}
|
|
93
|
+
} finally {
|
|
94
|
+
fs.closeSync(fd);
|
|
95
|
+
}
|
|
96
|
+
}
|
|
97
|
+
|
|
98
|
+
async function describeArtifact(file, onProgress) {
|
|
99
|
+
const stat = fs.statSync(file);
|
|
100
|
+
const { sha256 } = await sha256File(file, onProgress);
|
|
101
|
+
return {
|
|
102
|
+
file: path.basename(file),
|
|
103
|
+
bytes: stat.size,
|
|
104
|
+
sha256,
|
|
105
|
+
};
|
|
106
|
+
}
|
|
107
|
+
|
|
108
|
+
function synthesizeArchiveManifest(archive, version) {
|
|
109
|
+
const name = path.basename(archive);
|
|
110
|
+
const tag = name.match(/^larkit-truth-full-(.+)\.tar\.gz$/i)?.[1] || null;
|
|
111
|
+
return {
|
|
112
|
+
schemaVersion: PACK_SCHEMA_VERSION,
|
|
113
|
+
kind: PACK_KIND,
|
|
114
|
+
larkitVersion: version,
|
|
115
|
+
...(tag ? { packVersion: tag } : {}),
|
|
116
|
+
builtAt: null,
|
|
117
|
+
tiers: {
|
|
118
|
+
[TIER_NAME]: {
|
|
119
|
+
kind: "tar.gz",
|
|
120
|
+
file: name,
|
|
121
|
+
bytes: fs.statSync(archive).size,
|
|
122
|
+
},
|
|
123
|
+
},
|
|
124
|
+
};
|
|
125
|
+
}
|
|
126
|
+
|
|
127
|
+
/**
|
|
128
|
+
* Resolve a pack source into `{manifest, location}`.
|
|
129
|
+
*
|
|
130
|
+
* Accepted sources:
|
|
131
|
+
* - directory -> directory containing manifest.json
|
|
132
|
+
* - *.json / URL -> manifest (the channel API returns one too)
|
|
133
|
+
* - *.tar.gz / *.tgz -> a single unified archive
|
|
134
|
+
* - file:// URL -> local manifest or archive
|
|
135
|
+
*
|
|
136
|
+
* There is no bundled pack: every install downloads the unified archive from
|
|
137
|
+
* the channel (or from a local file / pack the caller passes in).
|
|
138
|
+
*/
|
|
139
|
+
export async function loadPackManifest(source) {
|
|
140
|
+
if (!source) {
|
|
141
|
+
throw new Error(
|
|
142
|
+
[
|
|
143
|
+
"no truth pack source.",
|
|
144
|
+
"",
|
|
145
|
+
"Pick one:",
|
|
146
|
+
" 1. larkit init --channel (download from the official truth channel)",
|
|
147
|
+
" 2. larkit init --pack=<manifest.json|dir|.tar.gz|https://...>",
|
|
148
|
+
" 3. LARKIT_PACK_MANIFEST=<url> larkit init",
|
|
149
|
+
" 4. larkit init --from-assets --deb=<feishu-linux.deb> --web-dump=<feishu-web-dump> (maintainer, Linux, slow)",
|
|
150
|
+
].join("\n"),
|
|
151
|
+
);
|
|
152
|
+
}
|
|
153
|
+
|
|
154
|
+
const raw = String(source);
|
|
155
|
+
const isRemote = isHttpUrl(raw) || isFileUrl(raw);
|
|
156
|
+
if (!isRemote) {
|
|
157
|
+
const resolved = path.resolve(raw);
|
|
158
|
+
if (fs.existsSync(resolved) && fs.statSync(resolved).isDirectory()) {
|
|
159
|
+
const file = path.join(resolved, "manifest.json");
|
|
160
|
+
if (!fs.existsSync(file)) {
|
|
161
|
+
throw new Error(`no manifest.json in pack directory: ${resolved}`);
|
|
162
|
+
}
|
|
163
|
+
return {
|
|
164
|
+
manifest: validateManifest(readJson(file, null), file),
|
|
165
|
+
location: { type: "file", base: resolved },
|
|
166
|
+
manifestFile: file,
|
|
167
|
+
};
|
|
168
|
+
}
|
|
169
|
+
if (/\.(tar\.gz|tgz)$/i.test(resolved)) {
|
|
170
|
+
return {
|
|
171
|
+
manifest: validateManifest(
|
|
172
|
+
synthesizeArchiveManifest(resolved, packageVersion()),
|
|
173
|
+
resolved,
|
|
174
|
+
),
|
|
175
|
+
location: { type: "file", base: path.dirname(resolved) },
|
|
176
|
+
manifestFile: null,
|
|
177
|
+
};
|
|
178
|
+
}
|
|
179
|
+
if (!fs.existsSync(resolved)) {
|
|
180
|
+
throw new Error(`pack not found: ${resolved}`);
|
|
181
|
+
}
|
|
182
|
+
return {
|
|
183
|
+
manifest: validateManifest(readJson(resolved, null), resolved),
|
|
184
|
+
location: { type: "file", base: path.dirname(resolved) },
|
|
185
|
+
manifestFile: resolved,
|
|
186
|
+
};
|
|
187
|
+
}
|
|
188
|
+
|
|
189
|
+
const pathname = isFileUrl(raw) ? fileURLToPath(raw) : new URL(raw).pathname;
|
|
190
|
+
if (/\.(tar\.gz|tgz)$/i.test(pathname)) {
|
|
191
|
+
if (isFileUrl(raw)) {
|
|
192
|
+
return {
|
|
193
|
+
manifest: validateManifest(
|
|
194
|
+
synthesizeArchiveManifest(pathname, packageVersion()),
|
|
195
|
+
pathname,
|
|
196
|
+
),
|
|
197
|
+
location: { type: "file", base: path.dirname(pathname) },
|
|
198
|
+
manifestFile: null,
|
|
199
|
+
};
|
|
200
|
+
}
|
|
201
|
+
return {
|
|
202
|
+
manifest: validateManifest(
|
|
203
|
+
{
|
|
204
|
+
schemaVersion: PACK_SCHEMA_VERSION,
|
|
205
|
+
kind: PACK_KIND,
|
|
206
|
+
larkitVersion: packageVersion(),
|
|
207
|
+
builtAt: null,
|
|
208
|
+
tiers: {
|
|
209
|
+
[TIER_NAME]: { kind: "tar.gz", file: pathname.split("/").pop() },
|
|
210
|
+
},
|
|
211
|
+
},
|
|
212
|
+
raw,
|
|
213
|
+
),
|
|
214
|
+
location: { type: "url", base: new URL(".", raw).href },
|
|
215
|
+
manifestFile: null,
|
|
216
|
+
};
|
|
217
|
+
}
|
|
218
|
+
|
|
219
|
+
const manifest = validateManifest(await fetchJson(raw), raw);
|
|
220
|
+
return {
|
|
221
|
+
manifest,
|
|
222
|
+
location: {
|
|
223
|
+
type: isHttpUrl(raw) ? "url" : "file",
|
|
224
|
+
base: isHttpUrl(raw) ? new URL(".", raw).href : path.dirname(fileURLToPath(raw)),
|
|
225
|
+
},
|
|
226
|
+
manifestFile: raw,
|
|
227
|
+
};
|
|
228
|
+
}
|
|
229
|
+
|
|
230
|
+
function validateManifest(manifest, source) {
|
|
231
|
+
if (!manifest || typeof manifest !== "object") {
|
|
232
|
+
throw new Error(`invalid pack manifest: ${source}`);
|
|
233
|
+
}
|
|
234
|
+
if (manifest.schemaVersion !== PACK_SCHEMA_VERSION) {
|
|
235
|
+
throw new Error(
|
|
236
|
+
`unsupported pack schema ${manifest.schemaVersion} (expected ${PACK_SCHEMA_VERSION}): ${source}`,
|
|
237
|
+
);
|
|
238
|
+
}
|
|
239
|
+
if (manifest.kind && manifest.kind !== PACK_KIND) {
|
|
240
|
+
throw new Error(`not a larkit truth pack: ${source}`);
|
|
241
|
+
}
|
|
242
|
+
if (!manifest.tiers?.[TIER_NAME]) {
|
|
243
|
+
throw new Error(
|
|
244
|
+
`pack manifest has no "${TIER_NAME}" tier: ${source}\n` +
|
|
245
|
+
"This looks like a legacy core/code pack; larkit now uses a single unified archive.",
|
|
246
|
+
);
|
|
247
|
+
}
|
|
248
|
+
return manifest;
|
|
249
|
+
}
|
|
250
|
+
|
|
251
|
+
function resolveArtifact(tier, location) {
|
|
252
|
+
if (tier.url) return tier.url;
|
|
253
|
+
if (!tier.file) throw new Error("pack tier has neither file nor url");
|
|
254
|
+
if (location.type === "url") {
|
|
255
|
+
return new URL(tier.file, location.base).href;
|
|
256
|
+
}
|
|
257
|
+
const root = path.resolve(location.base);
|
|
258
|
+
const resolved = path.resolve(root, tier.file);
|
|
259
|
+
if (
|
|
260
|
+
!path.isAbsolute(tier.file) &&
|
|
261
|
+
resolved !== root &&
|
|
262
|
+
!resolved.startsWith(root + path.sep)
|
|
263
|
+
) {
|
|
264
|
+
throw new Error(`pack artifact path escapes the manifest directory: ${tier.file}`);
|
|
265
|
+
}
|
|
266
|
+
return resolved;
|
|
267
|
+
}
|
|
268
|
+
|
|
269
|
+
export function mergeSeeds(seedDir, targetDir, previous = {}) {
|
|
270
|
+
const seeds = { ...previous };
|
|
271
|
+
if (!fs.existsSync(seedDir)) return seeds;
|
|
272
|
+
ensureDir(targetDir);
|
|
273
|
+
for (const name of fs.readdirSync(seedDir)) {
|
|
274
|
+
if (!name.endsWith(".md")) continue;
|
|
275
|
+
const source = path.join(seedDir, name);
|
|
276
|
+
if (!fs.statSync(source).isFile()) continue;
|
|
277
|
+
const dest = path.join(targetDir, name);
|
|
278
|
+
const next = sha256FileSync(source);
|
|
279
|
+
const sameFile = path.resolve(source) === path.resolve(dest);
|
|
280
|
+
if (!fs.existsSync(dest)) {
|
|
281
|
+
fs.copyFileSync(source, dest);
|
|
282
|
+
seeds[name] = next;
|
|
283
|
+
continue;
|
|
284
|
+
}
|
|
285
|
+
const current = sha256FileSync(dest);
|
|
286
|
+
const previousHash = previous[name];
|
|
287
|
+
if (previousHash && current === previousHash) {
|
|
288
|
+
if (!sameFile) fs.copyFileSync(source, dest);
|
|
289
|
+
seeds[name] = next;
|
|
290
|
+
} else if (!previousHash && current === next) {
|
|
291
|
+
seeds[name] = next;
|
|
292
|
+
}
|
|
293
|
+
// Otherwise the user edited the seed; keep their version.
|
|
294
|
+
}
|
|
295
|
+
return seeds;
|
|
296
|
+
}
|
|
297
|
+
|
|
298
|
+
function readStats() {
|
|
299
|
+
const meta = truthMeta();
|
|
300
|
+
return {
|
|
301
|
+
protocolCatalog: meta.protocolCatalog ?? null,
|
|
302
|
+
codeAnchors: meta.codeAnchors ?? null,
|
|
303
|
+
verifiedSections: meta.verifiedSections ?? null,
|
|
304
|
+
protocolDocs: meta.protocolDocs ?? null,
|
|
305
|
+
generatedAt: meta.generatedAt || null,
|
|
306
|
+
};
|
|
307
|
+
}
|
|
308
|
+
|
|
309
|
+
function artifactCachePath(tier, source) {
|
|
310
|
+
const base = tier.file || path.basename(new URL(source).pathname);
|
|
311
|
+
const prefix = tier.sha256 ? `${tier.sha256.slice(0, 16)}-` : "";
|
|
312
|
+
return path.join(DOWNLOAD_DIR, `${prefix}${base}`);
|
|
313
|
+
}
|
|
314
|
+
|
|
315
|
+
/**
|
|
316
|
+
* Reuse an already-downloaded archive when its checksum still matches, so a
|
|
317
|
+
* forced re-install (or an interrupted run) never pulls 161 MB twice.
|
|
318
|
+
*/
|
|
319
|
+
async function ensureArchive(tier, location, { quiet, reporter }) {
|
|
320
|
+
const source = resolveArtifact(tier, location);
|
|
321
|
+
if (!isHttpUrl(source)) {
|
|
322
|
+
if (!fs.existsSync(source)) {
|
|
323
|
+
throw new Error(
|
|
324
|
+
`pack archive not found: ${source}\nPublish it with --base-url=... or pass --pack=<manifest>`,
|
|
325
|
+
);
|
|
326
|
+
}
|
|
327
|
+
if (tier.sha256) {
|
|
328
|
+
const actual = sha256FileSync(source);
|
|
329
|
+
if (actual !== tier.sha256) throw new Error(`pack checksum mismatch for ${source}`);
|
|
330
|
+
}
|
|
331
|
+
return source;
|
|
332
|
+
}
|
|
333
|
+
const cached = artifactCachePath(tier, source);
|
|
334
|
+
if (tier.sha256 && fs.existsSync(cached) && sha256FileSync(cached) === tier.sha256) {
|
|
335
|
+
if (!quiet) log(`[pack] reuse cached ${path.basename(cached)}`);
|
|
336
|
+
return cached;
|
|
337
|
+
}
|
|
338
|
+
await downloadFile(source, cached, {
|
|
339
|
+
expectedSha256: tier.sha256,
|
|
340
|
+
expectedBytes: tier.bytes,
|
|
341
|
+
quiet,
|
|
342
|
+
onProgress: (received, total) => reporter?.progress?.("truth", received, total),
|
|
343
|
+
});
|
|
344
|
+
return cached;
|
|
345
|
+
}
|
|
346
|
+
|
|
347
|
+
/**
|
|
348
|
+
* Install the truth source: one archive carrying truth.db, the verified seeds
|
|
349
|
+
* and the formatted code corpus. There is nothing optional to select — the
|
|
350
|
+
* corpus is what makes `read_slice` work, so it always comes down.
|
|
351
|
+
*/
|
|
352
|
+
export async function installTruthPack({
|
|
353
|
+
source,
|
|
354
|
+
force = false,
|
|
355
|
+
quiet = false,
|
|
356
|
+
reporter,
|
|
357
|
+
state = {},
|
|
358
|
+
} = {}) {
|
|
359
|
+
const { manifest, location, manifestFile } = await loadPackManifest(source);
|
|
360
|
+
const larkitVersion = manifest.larkitVersion || manifest.version || packageVersion();
|
|
361
|
+
const packVersion = manifest.packVersion || larkitVersion;
|
|
362
|
+
const tier = manifest.tiers[TIER_NAME];
|
|
363
|
+
const packSha = tier.sha256 || `${TIER_NAME}:${tier.file}`;
|
|
364
|
+
const result = {
|
|
365
|
+
manifest,
|
|
366
|
+
manifest_file: manifestFile,
|
|
367
|
+
location: location.type,
|
|
368
|
+
version: packVersion,
|
|
369
|
+
packVersion,
|
|
370
|
+
larkitVersion,
|
|
371
|
+
packSha,
|
|
372
|
+
status: null,
|
|
373
|
+
source: null,
|
|
374
|
+
seeds: {},
|
|
375
|
+
stats: readStats(),
|
|
376
|
+
};
|
|
377
|
+
|
|
378
|
+
if (!force && fs.existsSync(TRUTH_DB) && state?.truth?.packSha === packSha) {
|
|
379
|
+
return {
|
|
380
|
+
...result,
|
|
381
|
+
status: "present",
|
|
382
|
+
source: state.truth.source,
|
|
383
|
+
seeds: state.truth.seeds || {},
|
|
384
|
+
skipped: true,
|
|
385
|
+
};
|
|
386
|
+
}
|
|
387
|
+
|
|
388
|
+
const archive = await ensureArchive(tier, location, { quiet, reporter });
|
|
389
|
+
ensureDir(DATA_DIR);
|
|
390
|
+
const tmp = fs.mkdtempSync(path.join(DATA_DIR, ".pack-full-"));
|
|
391
|
+
try {
|
|
392
|
+
const files = await extractTarGz(archive, tmp, {
|
|
393
|
+
onWarning: (message) => reporter?.warn(message),
|
|
394
|
+
onProgress: (bytes) => reporter?.progress?.("truth", bytes, tier.unpackedBytes),
|
|
395
|
+
});
|
|
396
|
+
|
|
397
|
+
const incoming = path.join(tmp, "truth.db");
|
|
398
|
+
if (!fs.existsSync(incoming)) {
|
|
399
|
+
throw new Error("unified pack does not contain truth.db");
|
|
400
|
+
}
|
|
401
|
+
assertSqlite(incoming);
|
|
402
|
+
fs.rmSync(TRUTH_DB, { force: true });
|
|
403
|
+
for (const suffix of ["-wal", "-shm"]) fs.rmSync(`${TRUTH_DB}${suffix}`, { force: true });
|
|
404
|
+
fs.renameSync(incoming, TRUTH_DB);
|
|
405
|
+
|
|
406
|
+
const meta = path.join(tmp, "truth.meta.json");
|
|
407
|
+
if (fs.existsSync(meta)) fs.copyFileSync(meta, TRUTH_META);
|
|
408
|
+
|
|
409
|
+
const seeds = mergeSeeds(
|
|
410
|
+
path.join(tmp, "verified_truths"),
|
|
411
|
+
VERIFIED_DIR,
|
|
412
|
+
!force ? state?.truth?.seeds || {} : {},
|
|
413
|
+
);
|
|
414
|
+
|
|
415
|
+
const incomingCode = path.join(tmp, "formatted");
|
|
416
|
+
if (!fs.existsSync(incomingCode)) {
|
|
417
|
+
throw new Error("unified pack does not contain formatted/; rebuild it with `larkit pack build`");
|
|
418
|
+
}
|
|
419
|
+
const backup = `${FORMATTED_DIR}.old-${Date.now()}`;
|
|
420
|
+
if (fs.existsSync(FORMATTED_DIR)) fs.renameSync(FORMATTED_DIR, backup);
|
|
421
|
+
fs.renameSync(incomingCode, FORMATTED_DIR);
|
|
422
|
+
fs.rmSync(backup, { recursive: true, force: true });
|
|
423
|
+
|
|
424
|
+
return {
|
|
425
|
+
...result,
|
|
426
|
+
status: "installed",
|
|
427
|
+
source: location.type === "url" ? "download" : "archive",
|
|
428
|
+
archive,
|
|
429
|
+
seeds,
|
|
430
|
+
files: files.files,
|
|
431
|
+
bytes: files.bytes,
|
|
432
|
+
stats: readStats(),
|
|
433
|
+
skipped: false,
|
|
434
|
+
};
|
|
435
|
+
} finally {
|
|
436
|
+
fs.rmSync(tmp, { recursive: true, force: true });
|
|
437
|
+
}
|
|
438
|
+
}
|
|
439
|
+
|
|
440
|
+
/**
|
|
441
|
+
* Package-maintainer helper: snapshot DATA_DIR + VERIFIED_DIR + FORMATTED_DIR
|
|
442
|
+
* into one versioned archive and a manifest that points at it.
|
|
443
|
+
*/
|
|
444
|
+
export async function buildTruthPack({
|
|
445
|
+
outDir = path.join(PKG_ROOT, "dist", "truth-pack"),
|
|
446
|
+
dataDir = DATA_DIR,
|
|
447
|
+
verifiedDir = VERIFIED_DIR,
|
|
448
|
+
baseUrl = "",
|
|
449
|
+
version = packageVersion(),
|
|
450
|
+
tag = "",
|
|
451
|
+
onProgress,
|
|
452
|
+
} = {}) {
|
|
453
|
+
const packTag = normalizeTag(tag) || version;
|
|
454
|
+
const truthDb = path.join(dataDir, "truth.db");
|
|
455
|
+
if (!fs.existsSync(truthDb)) {
|
|
456
|
+
throw new Error(`missing ${truthDb}; run "larkit index" or "larkit init --from-assets" first`);
|
|
457
|
+
}
|
|
458
|
+
const metaPath = path.join(dataDir, "truth.meta.json");
|
|
459
|
+
ensureDir(outDir);
|
|
460
|
+
const seeds = markdownFiles(verifiedDir);
|
|
461
|
+
|
|
462
|
+
const formatted = path.join(dataDir, "formatted");
|
|
463
|
+
const files = walkFiles(formatted, { skipDirs: new Set([".git", "node_modules"]) });
|
|
464
|
+
if (!files.length) throw new Error(`missing ${formatted}; cannot build the truth pack`);
|
|
465
|
+
const formattedBytes = files.reduce((sum, file) => sum + file.size, 0);
|
|
466
|
+
log(
|
|
467
|
+
`[pack] truth: truth.db + ${seeds.length} seeds + ${files.length} code files (${fileSizeMiB(formattedBytes)})`,
|
|
468
|
+
);
|
|
469
|
+
|
|
470
|
+
const entries = [
|
|
471
|
+
{ file: truthDb, tarPath: "truth.db" },
|
|
472
|
+
...(fs.existsSync(metaPath) ? [{ file: metaPath, tarPath: "truth.meta.json" }] : []),
|
|
473
|
+
...seeds.map((file) => ({
|
|
474
|
+
file,
|
|
475
|
+
tarPath: `verified_truths/${path.basename(file)}`,
|
|
476
|
+
})),
|
|
477
|
+
...files.map((entry) => ({
|
|
478
|
+
file: entry.path,
|
|
479
|
+
tarPath: `formatted/${path.relative(formatted, entry.path).split(path.sep).join("/")}`,
|
|
480
|
+
})),
|
|
481
|
+
];
|
|
482
|
+
|
|
483
|
+
const file = tierFileName(packTag);
|
|
484
|
+
const archive = path.join(outDir, file);
|
|
485
|
+
await createTarGz(archive, { entries, onProgress });
|
|
486
|
+
const full = await describeArtifact(archive, onProgress);
|
|
487
|
+
full.kind = "tar.gz";
|
|
488
|
+
full.unpackedBytes =
|
|
489
|
+
fs.statSync(truthDb).size +
|
|
490
|
+
(fs.existsSync(metaPath) ? fs.statSync(metaPath).size : 0) +
|
|
491
|
+
seeds.reduce((sum, seed) => sum + fs.statSync(seed).size, 0) +
|
|
492
|
+
formattedBytes;
|
|
493
|
+
full.codeFiles = files.length;
|
|
494
|
+
if (baseUrl) full.url = `${baseUrl.replace(/\/+$/, "")}/${file}`;
|
|
495
|
+
|
|
496
|
+
const manifest = {
|
|
497
|
+
schemaVersion: PACK_SCHEMA_VERSION,
|
|
498
|
+
kind: PACK_KIND,
|
|
499
|
+
larkitVersion: version,
|
|
500
|
+
packVersion: packTag,
|
|
501
|
+
builtAt: new Date().toISOString(),
|
|
502
|
+
clientVersion: readJson(path.join(dataDir, "raw", "manifest.extract.json"), {})
|
|
503
|
+
.clientVersion || null,
|
|
504
|
+
stats: readStats(),
|
|
505
|
+
tiers: { [TIER_NAME]: full },
|
|
506
|
+
};
|
|
507
|
+
const manifestFile = path.join(outDir, "manifest.json");
|
|
508
|
+
writeJson(manifestFile, manifest);
|
|
509
|
+
return { manifest, manifestFile, files: { full: archive } };
|
|
510
|
+
}
|
|
511
|
+
|
|
512
|
+
/** Verify local pack artifacts referenced by a manifest. */
|
|
513
|
+
export async function verifyPack(source) {
|
|
514
|
+
const { manifest, location, manifestFile } = await loadPackManifest(source);
|
|
515
|
+
const checked = [];
|
|
516
|
+
for (const [name, tier] of Object.entries(manifest.tiers || {})) {
|
|
517
|
+
const artifact = resolveArtifact(tier, location);
|
|
518
|
+
if (isHttpUrl(artifact)) {
|
|
519
|
+
checked.push({ tier: name, url: artifact, ok: true, remote: true });
|
|
520
|
+
continue;
|
|
521
|
+
}
|
|
522
|
+
if (!fs.existsSync(artifact)) {
|
|
523
|
+
checked.push({ tier: name, file: artifact, ok: false, error: "missing" });
|
|
524
|
+
continue;
|
|
525
|
+
}
|
|
526
|
+
const stat = fs.statSync(artifact);
|
|
527
|
+
const actual = sha256FileSync(artifact);
|
|
528
|
+
checked.push({
|
|
529
|
+
tier: name,
|
|
530
|
+
file: artifact,
|
|
531
|
+
bytes: stat.size,
|
|
532
|
+
ok: (!tier.sha256 || actual === tier.sha256) && (!tier.bytes || stat.size === tier.bytes),
|
|
533
|
+
});
|
|
534
|
+
}
|
|
535
|
+
return {
|
|
536
|
+
ok: checked.every((item) => item.ok),
|
|
537
|
+
manifest_file: manifestFile,
|
|
538
|
+
version: manifest.packVersion || manifest.larkitVersion,
|
|
539
|
+
tiers: checked,
|
|
540
|
+
};
|
|
541
|
+
}
|
|
@@ -0,0 +1,62 @@
|
|
|
1
|
+
import process from "node:process";
|
|
2
|
+
import { fileSizeMiB } from "../shared/util.js";
|
|
3
|
+
|
|
4
|
+
const paint = (enabled) => (code, text) =>
|
|
5
|
+
enabled ? `\x1b[${code}m${text}\x1b[0m` : String(text);
|
|
6
|
+
|
|
7
|
+
/**
|
|
8
|
+
* Human output for interactive `larkit init/status`. In `--json` mode every
|
|
9
|
+
* human line goes to stderr so stdout stays a single JSON document.
|
|
10
|
+
*/
|
|
11
|
+
export function createReporter({ json = false, quiet = false } = {}) {
|
|
12
|
+
const humanStream = json ? process.stderr : process.stdout;
|
|
13
|
+
const color = paint(Boolean(humanStream.isTTY) && !process.env.NO_COLOR);
|
|
14
|
+
const write = (line = "") => humanStream.write(`${line}\n`);
|
|
15
|
+
const lastProgress = new Map();
|
|
16
|
+
let stepNumber = 0;
|
|
17
|
+
|
|
18
|
+
return {
|
|
19
|
+
json,
|
|
20
|
+
quiet,
|
|
21
|
+
line: write,
|
|
22
|
+
header(title, subtitle) {
|
|
23
|
+
if (quiet) return;
|
|
24
|
+
write(`${color("1", title)}${subtitle ? ` ${color("2", subtitle)}` : ""}`);
|
|
25
|
+
},
|
|
26
|
+
step(title) {
|
|
27
|
+
stepNumber++;
|
|
28
|
+
if (!quiet) write(`\n${color("36", `[${stepNumber}]`)} ${color("1", title)}`);
|
|
29
|
+
},
|
|
30
|
+
info(message) {
|
|
31
|
+
if (!quiet) write(` ${message}`);
|
|
32
|
+
},
|
|
33
|
+
ok(message) {
|
|
34
|
+
if (!quiet) write(` ${color("32", "✓")} ${message}`);
|
|
35
|
+
},
|
|
36
|
+
detail(label, value) {
|
|
37
|
+
if (!quiet) write(` ${color("2", String(label).padEnd(12))} ${value}`);
|
|
38
|
+
},
|
|
39
|
+
warn(message) {
|
|
40
|
+
if (!quiet) process.stderr.write(` ${color("33", "!")} ${message}\n`);
|
|
41
|
+
},
|
|
42
|
+
error(message) {
|
|
43
|
+
process.stderr.write(` ${color("31", "✗")} ${message}\n`);
|
|
44
|
+
},
|
|
45
|
+
progress(kind, received, total) {
|
|
46
|
+
if (quiet || json || !humanStream.isTTY) return;
|
|
47
|
+
const now = Date.now();
|
|
48
|
+
if (!total && now - (lastProgress.get(kind) || 0) < 2000) return;
|
|
49
|
+
if (total && now - (lastProgress.get(kind) || 0) < 250) return;
|
|
50
|
+
lastProgress.set(kind, now);
|
|
51
|
+
const percent = total ? Math.min(100, Math.round((received / total) * 100)) : null;
|
|
52
|
+
const text = total
|
|
53
|
+
? `${percent}% ${fileSizeMiB(received)} / ${fileSizeMiB(total)}`
|
|
54
|
+
: fileSizeMiB(received);
|
|
55
|
+
process.stderr.write(` ${color("2", kind.padEnd(6))} ${text}\x1b[K\r`);
|
|
56
|
+
if (total && received >= total) process.stderr.write("\n");
|
|
57
|
+
},
|
|
58
|
+
result(value) {
|
|
59
|
+
if (json) process.stdout.write(`${JSON.stringify(value, null, 2)}\n`);
|
|
60
|
+
},
|
|
61
|
+
};
|
|
62
|
+
}
|