@sleepwalkerai/cli 0.1.0 → 0.2.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +24 -12
- package/package.json +1 -1
- package/src/cli.js +83 -1
- package/src/okf.js +430 -0
- package/src/theme.js +3 -0
package/README.md
CHANGED
|
@@ -11,7 +11,9 @@ Public package:
|
|
|
11
11
|
@sleepwalkerai/cli
|
|
12
12
|
```
|
|
13
13
|
|
|
14
|
-
|
|
14
|
+
The package remains private in this monorepo so it cannot be published from
|
|
15
|
+
the product repository by mistake. Public releases happen from the
|
|
16
|
+
`followanton/sleepwalker` developer repository.
|
|
15
17
|
|
|
16
18
|
## Install
|
|
17
19
|
|
|
@@ -21,6 +23,7 @@ sleepwalker init
|
|
|
21
23
|
sleepwalker auth key set sw_api_live_...
|
|
22
24
|
sleepwalker menu
|
|
23
25
|
sleepwalker doctor
|
|
26
|
+
sleepwalker commands
|
|
24
27
|
```
|
|
25
28
|
|
|
26
29
|
One-off usage:
|
|
@@ -34,32 +37,31 @@ npx @sleepwalkerai/cli doctor
|
|
|
34
37
|
From this repository:
|
|
35
38
|
|
|
36
39
|
```bash
|
|
37
|
-
|
|
38
|
-
node
|
|
39
|
-
node
|
|
40
|
-
node
|
|
41
|
-
node ./bin/sleepwalker.js commands
|
|
40
|
+
node sleepwalker-cli/bin/sleepwalker.js
|
|
41
|
+
node sleepwalker-cli/bin/sleepwalker.js menu
|
|
42
|
+
node sleepwalker-cli/bin/sleepwalker.js init
|
|
43
|
+
node sleepwalker-cli/bin/sleepwalker.js commands
|
|
42
44
|
```
|
|
43
45
|
|
|
44
46
|
With an API key:
|
|
45
47
|
|
|
46
48
|
```bash
|
|
47
49
|
export SLEEPWALKER_API_KEY=sw_api_live_...
|
|
48
|
-
node
|
|
50
|
+
node sleepwalker-cli/bin/sleepwalker.js credits
|
|
49
51
|
```
|
|
50
52
|
|
|
51
53
|
Or store the key locally:
|
|
52
54
|
|
|
53
55
|
```bash
|
|
54
|
-
node
|
|
55
|
-
node
|
|
56
|
+
node sleepwalker-cli/bin/sleepwalker.js auth key set sw_api_live_...
|
|
57
|
+
node sleepwalker-cli/bin/sleepwalker.js doctor
|
|
56
58
|
```
|
|
57
59
|
|
|
58
60
|
To point the CLI at a non-production API base:
|
|
59
61
|
|
|
60
62
|
```bash
|
|
61
|
-
node
|
|
62
|
-
node
|
|
63
|
+
node sleepwalker-cli/bin/sleepwalker.js config set api-base-url https://api.sleepwalker.ai
|
|
64
|
+
node sleepwalker-cli/bin/sleepwalker.js config show
|
|
63
65
|
```
|
|
64
66
|
|
|
65
67
|
## Examples
|
|
@@ -74,8 +76,18 @@ sleepwalker visibility run https://www.sleepwalker.ai --brand Sleepwalker --prom
|
|
|
74
76
|
sleepwalker ci score https://www.sleepwalker.ai
|
|
75
77
|
sleepwalker ci run https://www.sleepwalker.ai --depth full --watch
|
|
76
78
|
sleepwalker activity list
|
|
79
|
+
sleepwalker okf export https://www.sleepwalker.ai
|
|
77
80
|
```
|
|
78
81
|
|
|
82
|
+
`okf export` is free, open source (MIT, like the rest of this CLI), and runs
|
|
83
|
+
entirely on your machine — it fetches the page and writes an
|
|
84
|
+
[Open Knowledge Format](https://github.com/GoogleCloudPlatform/knowledge-catalog/tree/main/okf)
|
|
85
|
+
bundle (agent-ready markdown) to a local directory. No account, no API key, no
|
|
86
|
+
credits. Use `--out <dir>` to choose the destination and `--force` to overwrite.
|
|
87
|
+
Hostile page content is sanitized (control characters, ANSI escapes, and bidi
|
|
88
|
+
overrides are stripped), fetches time out after 30 seconds, and oversized pages
|
|
89
|
+
are truncated with a note in `log.md`.
|
|
90
|
+
|
|
79
91
|
Add `--json` to print raw API responses.
|
|
80
92
|
|
|
81
93
|
For retryable scripts, pass your own idempotency key on persisted run creation:
|
|
@@ -130,5 +142,5 @@ or `FORCE_COLOR=1` to force it in supported terminals.
|
|
|
130
142
|
If your current shell exports `NO_COLOR`, remove it while previewing:
|
|
131
143
|
|
|
132
144
|
```bash
|
|
133
|
-
env -u NO_COLOR FORCE_COLOR=1 node
|
|
145
|
+
env -u NO_COLOR FORCE_COLOR=1 node sleepwalker-cli/bin/sleepwalker.js --help
|
|
134
146
|
```
|
package/package.json
CHANGED
package/src/cli.js
CHANGED
|
@@ -18,9 +18,10 @@ import {
|
|
|
18
18
|
readLinesFromFile,
|
|
19
19
|
} from "./args.js";
|
|
20
20
|
import { printJson, printKeyValue, printList, printNextCommands, printRunSummary } from "./format.js";
|
|
21
|
+
import { buildBundle, defaultOutDir, writeBundle, OKF_USER_AGENT } from "./okf.js";
|
|
21
22
|
import { createTheme, renderCommandsHelp, renderHelp, sanitizeTerminalText, styleStatus } from "./theme.js";
|
|
22
23
|
|
|
23
|
-
const VERSION = "0.
|
|
24
|
+
const VERSION = "0.2.0";
|
|
24
25
|
const DEFAULT_POLL_INTERVAL_MS = 5000;
|
|
25
26
|
const DEFAULT_MAX_WAIT_SECONDS = 900;
|
|
26
27
|
const TERMINAL_RUN_STATUSES = new Set([
|
|
@@ -1105,6 +1106,85 @@ async function handleCi(args, flags, io) {
|
|
|
1105
1106
|
throw makeError("Unknown ci command. Run `sleepwalker --help`.");
|
|
1106
1107
|
}
|
|
1107
1108
|
|
|
1109
|
+
async function handleOkf(args, flags, io) {
|
|
1110
|
+
const theme = io.theme;
|
|
1111
|
+
const usage = "Usage: sleepwalker okf export <url> [--out <dir>] [--force]";
|
|
1112
|
+
if (args[0] !== "export") {
|
|
1113
|
+
throw makeError(usage);
|
|
1114
|
+
}
|
|
1115
|
+
const url = requireArg(args[1] || flagString(flags, "url", ""), usage);
|
|
1116
|
+
const fetchImpl = io.fetch || globalThis.fetch;
|
|
1117
|
+
if (typeof fetchImpl !== "function") {
|
|
1118
|
+
throw makeError("This command needs the built-in fetch (Node >= 18).");
|
|
1119
|
+
}
|
|
1120
|
+
|
|
1121
|
+
let response;
|
|
1122
|
+
try {
|
|
1123
|
+
response = await fetchImpl(url, {
|
|
1124
|
+
redirect: "follow",
|
|
1125
|
+
headers: { "user-agent": OKF_USER_AGENT, accept: "text/html,application/xhtml+xml" },
|
|
1126
|
+
// A hanging or slow-dripping server should fail the command, not stall it.
|
|
1127
|
+
signal: typeof AbortSignal?.timeout === "function" ? AbortSignal.timeout(30_000) : undefined,
|
|
1128
|
+
});
|
|
1129
|
+
} catch (error) {
|
|
1130
|
+
const reason = error?.name === "TimeoutError" ? "timed out after 30s" : error.message;
|
|
1131
|
+
throw makeError(`Could not fetch ${url}: ${reason}`);
|
|
1132
|
+
}
|
|
1133
|
+
if (!response.ok) {
|
|
1134
|
+
throw makeError(`Fetch failed for ${url}: HTTP ${response.status}.`);
|
|
1135
|
+
}
|
|
1136
|
+
const finalUrl = response.url || url;
|
|
1137
|
+
const extraNotes = [];
|
|
1138
|
+
const contentType = String(response.headers?.get?.("content-type") || "");
|
|
1139
|
+
if (contentType && !/html|xml/i.test(contentType)) {
|
|
1140
|
+
extraNotes.push(`response content-type was ${contentType.split(";")[0].trim()}; extraction may be unreliable`);
|
|
1141
|
+
}
|
|
1142
|
+
const MAX_HTML_CHARS = 5_000_000;
|
|
1143
|
+
let html = await response.text();
|
|
1144
|
+
if (html.length > MAX_HTML_CHARS) {
|
|
1145
|
+
html = html.slice(0, MAX_HTML_CHARS);
|
|
1146
|
+
extraNotes.push(`page exceeded ${MAX_HTML_CHARS} characters; extraction used the truncated beginning`);
|
|
1147
|
+
}
|
|
1148
|
+
const now = new Date().toISOString();
|
|
1149
|
+
const { files, summary } = buildBundle({ url: finalUrl, html, now, cliVersion: VERSION, extraNotes });
|
|
1150
|
+
const outDir = flagString(flags, "out", "") || defaultOutDir(finalUrl);
|
|
1151
|
+
const { dir } = await writeBundle(outDir, files, { force: flagBool(flags, "force", false) });
|
|
1152
|
+
|
|
1153
|
+
const payload = {
|
|
1154
|
+
url: finalUrl,
|
|
1155
|
+
out: dir,
|
|
1156
|
+
okf_version: "0.1",
|
|
1157
|
+
credits: 0,
|
|
1158
|
+
concepts: summary.conceptCount,
|
|
1159
|
+
files: summary.files,
|
|
1160
|
+
notes: summary.notes,
|
|
1161
|
+
};
|
|
1162
|
+
output(io.stdout, flags, payload, (data) => {
|
|
1163
|
+
io.stdout.write(`${theme.accent("OKF bundle created")}\n\n`);
|
|
1164
|
+
printKeyValue(
|
|
1165
|
+
io.stdout,
|
|
1166
|
+
[
|
|
1167
|
+
["URL", theme.info(data.url)],
|
|
1168
|
+
["Output", data.out],
|
|
1169
|
+
["Files", String(data.files.length)],
|
|
1170
|
+
["Credits", "0 (ran locally — no account needed)"],
|
|
1171
|
+
],
|
|
1172
|
+
theme,
|
|
1173
|
+
);
|
|
1174
|
+
if (data.notes && data.notes.length) {
|
|
1175
|
+
io.stdout.write(`\n${theme.muted(`note: ${data.notes.join("; ")}`)}\n`);
|
|
1176
|
+
}
|
|
1177
|
+
printNextCommands(
|
|
1178
|
+
io.stdout,
|
|
1179
|
+
[
|
|
1180
|
+
`sleepwalker okf export ${shellQuote(data.url)} --json`,
|
|
1181
|
+
`sleepwalker ci score ${shellQuote(data.url)} # engine-grade analysis (account + credits)`,
|
|
1182
|
+
],
|
|
1183
|
+
theme,
|
|
1184
|
+
);
|
|
1185
|
+
});
|
|
1186
|
+
}
|
|
1187
|
+
|
|
1108
1188
|
export async function runCli(argv, io) {
|
|
1109
1189
|
const { positional, flags } = parseFlags(argv);
|
|
1110
1190
|
io.theme = createTheme({ env: io.env, stdout: io.stdout });
|
|
@@ -1151,6 +1231,8 @@ export async function runCli(argv, io) {
|
|
|
1151
1231
|
await handleVisibility(rest, flags, io);
|
|
1152
1232
|
} else if (command === "ci" || command === "content") {
|
|
1153
1233
|
await handleCi(rest, flags, io);
|
|
1234
|
+
} else if (command === "okf") {
|
|
1235
|
+
await handleOkf(rest, flags, io);
|
|
1154
1236
|
} else {
|
|
1155
1237
|
throw makeError(`Unknown command: ${command}. Run \`sleepwalker --help\`.`);
|
|
1156
1238
|
}
|
package/src/okf.js
ADDED
|
@@ -0,0 +1,430 @@
|
|
|
1
|
+
// Open Knowledge Format (OKF v0.1) export — local, dependency-free.
|
|
2
|
+
//
|
|
3
|
+
// This module holds the PURE transforms: HTML string -> OKF bundle files.
|
|
4
|
+
// It performs no network and no filesystem I/O (except writeBundle, which is
|
|
5
|
+
// kept separate so the transforms stay unit-testable). The CLI handler fetches
|
|
6
|
+
// the page and calls buildBundle(); nothing here touches the Sleepwalker API,
|
|
7
|
+
// so `okf export` costs zero credits and needs no account.
|
|
8
|
+
//
|
|
9
|
+
// Fidelity is intentionally "good enough" for a free tier. A future upgrade can
|
|
10
|
+
// swap the local extractor for readability/turndown behind the same interface.
|
|
11
|
+
|
|
12
|
+
import { mkdir, readdir, writeFile } from "node:fs/promises";
|
|
13
|
+
import path from "node:path";
|
|
14
|
+
|
|
15
|
+
import { sanitizeTerminalText } from "./theme.js";
|
|
16
|
+
|
|
17
|
+
// Bundles are "agent-ready markdown": hostile page bytes (control characters,
|
|
18
|
+
// ANSI escapes, bidi overrides — raw or entity-encoded) must never reach the
|
|
19
|
+
// emitted files, both so strict YAML consumers can parse the frontmatter and
|
|
20
|
+
// so cat-ing a bundle can't inject terminal escapes or Trojan-Source spoofing.
|
|
21
|
+
function sanitizeText(value) {
|
|
22
|
+
return sanitizeTerminalText(value);
|
|
23
|
+
}
|
|
24
|
+
|
|
25
|
+
export const OKF_VERSION = "0.1";
|
|
26
|
+
export const OKF_USER_AGENT =
|
|
27
|
+
"SleepwalkerCLI-OKF/0.1 (+https://github.com/followanton/sleepwalker)";
|
|
28
|
+
|
|
29
|
+
const RESERVED_FILENAMES = new Set(["index.md", "log.md"]);
|
|
30
|
+
|
|
31
|
+
// Blocks whose contents are never page content.
|
|
32
|
+
const NOISE_BLOCKS = [
|
|
33
|
+
"script",
|
|
34
|
+
"style",
|
|
35
|
+
"noscript",
|
|
36
|
+
"template",
|
|
37
|
+
"svg",
|
|
38
|
+
"head",
|
|
39
|
+
"nav",
|
|
40
|
+
"footer",
|
|
41
|
+
"header",
|
|
42
|
+
"aside",
|
|
43
|
+
"form",
|
|
44
|
+
"iframe",
|
|
45
|
+
];
|
|
46
|
+
|
|
47
|
+
const NAMED_ENTITIES = {
|
|
48
|
+
amp: "&",
|
|
49
|
+
lt: "<",
|
|
50
|
+
gt: ">",
|
|
51
|
+
quot: '"',
|
|
52
|
+
apos: "'",
|
|
53
|
+
"#39": "'",
|
|
54
|
+
nbsp: " ",
|
|
55
|
+
mdash: "—",
|
|
56
|
+
ndash: "–",
|
|
57
|
+
hellip: "…",
|
|
58
|
+
copy: "©",
|
|
59
|
+
reg: "®",
|
|
60
|
+
trade: "™",
|
|
61
|
+
rsquo: "’",
|
|
62
|
+
lsquo: "‘",
|
|
63
|
+
ldquo: "“",
|
|
64
|
+
rdquo: "”",
|
|
65
|
+
rarr: "→",
|
|
66
|
+
larr: "←",
|
|
67
|
+
uarr: "↑",
|
|
68
|
+
darr: "↓",
|
|
69
|
+
harr: "↔",
|
|
70
|
+
times: "×",
|
|
71
|
+
divide: "÷",
|
|
72
|
+
deg: "°",
|
|
73
|
+
plusmn: "±",
|
|
74
|
+
middot: "·",
|
|
75
|
+
bull: "•",
|
|
76
|
+
laquo: "«",
|
|
77
|
+
raquo: "»",
|
|
78
|
+
euro: "€",
|
|
79
|
+
pound: "£",
|
|
80
|
+
cent: "¢",
|
|
81
|
+
yen: "¥",
|
|
82
|
+
sect: "§",
|
|
83
|
+
};
|
|
84
|
+
|
|
85
|
+
export function decodeEntities(input) {
|
|
86
|
+
if (!input) return "";
|
|
87
|
+
return String(input).replace(/&(#x?[0-9a-fA-F]+|[a-zA-Z][a-zA-Z0-9]*);/g, (match, body) => {
|
|
88
|
+
if (body[0] === "#") {
|
|
89
|
+
const code =
|
|
90
|
+
body[1] === "x" || body[1] === "X"
|
|
91
|
+
? parseInt(body.slice(2), 16)
|
|
92
|
+
: parseInt(body.slice(1), 10);
|
|
93
|
+
if (Number.isFinite(code) && code > 0) {
|
|
94
|
+
try {
|
|
95
|
+
return String.fromCodePoint(code);
|
|
96
|
+
} catch {
|
|
97
|
+
return match;
|
|
98
|
+
}
|
|
99
|
+
}
|
|
100
|
+
return match;
|
|
101
|
+
}
|
|
102
|
+
const named = NAMED_ENTITIES[body] ?? NAMED_ENTITIES[body.toLowerCase()];
|
|
103
|
+
return named ?? match;
|
|
104
|
+
});
|
|
105
|
+
}
|
|
106
|
+
|
|
107
|
+
function stripTags(html) {
|
|
108
|
+
return html.replace(/<[^>]+>/g, "");
|
|
109
|
+
}
|
|
110
|
+
|
|
111
|
+
function removeBlock(html, tag) {
|
|
112
|
+
const re = new RegExp(`<${tag}\\b[^>]*>[\\s\\S]*?</${tag}>`, "gi");
|
|
113
|
+
return html.replace(re, " ");
|
|
114
|
+
}
|
|
115
|
+
|
|
116
|
+
function removeNoise(html) {
|
|
117
|
+
let out = html.replace(/<!--[\s\S]*?-->/g, " ");
|
|
118
|
+
for (const tag of NOISE_BLOCKS) out = removeBlock(out, tag);
|
|
119
|
+
// Drop any self-closing/dangling noise open tags left behind.
|
|
120
|
+
return out;
|
|
121
|
+
}
|
|
122
|
+
|
|
123
|
+
function collapseWhitespace(text) {
|
|
124
|
+
return text
|
|
125
|
+
.replace(/[ \t\f\v]+/g, " ")
|
|
126
|
+
.replace(/ *\n */g, "\n")
|
|
127
|
+
.replace(/\n{3,}/g, "\n\n")
|
|
128
|
+
.trim();
|
|
129
|
+
}
|
|
130
|
+
|
|
131
|
+
function firstMatch(html, re) {
|
|
132
|
+
const m = re.exec(html);
|
|
133
|
+
return m ? decodeEntities(stripTags(m[1]).trim()) : "";
|
|
134
|
+
}
|
|
135
|
+
|
|
136
|
+
// Read the content="" of the first <meta> tag whose name/property matches.
|
|
137
|
+
function metaContent(html, nameValue) {
|
|
138
|
+
const tagRe = /<meta\b[^>]*>/gi;
|
|
139
|
+
let tag;
|
|
140
|
+
while ((tag = tagRe.exec(html))) {
|
|
141
|
+
const raw = tag[0];
|
|
142
|
+
const nameM = /\b(?:name|property)\s*=\s*["']?([^"'>\s]+)/i.exec(raw);
|
|
143
|
+
if (nameM && nameM[1].toLowerCase() === nameValue.toLowerCase()) {
|
|
144
|
+
const contentM = /\bcontent\s*=\s*["']([\s\S]*?)["']/i.exec(raw);
|
|
145
|
+
if (contentM) return decodeEntities(contentM[1].trim());
|
|
146
|
+
}
|
|
147
|
+
}
|
|
148
|
+
return "";
|
|
149
|
+
}
|
|
150
|
+
|
|
151
|
+
function canonicalUrl(html) {
|
|
152
|
+
const linkRe = /<link\b[^>]*>/gi;
|
|
153
|
+
let tag;
|
|
154
|
+
while ((tag = linkRe.exec(html))) {
|
|
155
|
+
const raw = tag[0];
|
|
156
|
+
if (/\brel\s*=\s*["']?canonical/i.test(raw)) {
|
|
157
|
+
const hrefM = /\bhref\s*=\s*["']([^"']+)["']/i.exec(raw);
|
|
158
|
+
if (hrefM) return hrefM[1].trim();
|
|
159
|
+
}
|
|
160
|
+
}
|
|
161
|
+
return "";
|
|
162
|
+
}
|
|
163
|
+
|
|
164
|
+
// Pick the most content-ful region: prefer <main>/<article>, else <body>.
|
|
165
|
+
function mainRegion(html) {
|
|
166
|
+
for (const tag of ["main", "article"]) {
|
|
167
|
+
const re = new RegExp(`<${tag}\\b[^>]*>([\\s\\S]*?)</${tag}>`, "gi");
|
|
168
|
+
let best = "";
|
|
169
|
+
let m;
|
|
170
|
+
while ((m = re.exec(html))) {
|
|
171
|
+
if (m[1].length > best.length) best = m[1];
|
|
172
|
+
}
|
|
173
|
+
if (best.trim()) return best;
|
|
174
|
+
}
|
|
175
|
+
const body = /<body\b[^>]*>([\s\S]*?)<\/body>/i.exec(html);
|
|
176
|
+
return body ? body[1] : html;
|
|
177
|
+
}
|
|
178
|
+
|
|
179
|
+
// Lightweight HTML -> markdown for the pieces that matter to a reader/agent:
|
|
180
|
+
// headings, paragraphs, list items, links. Everything else becomes plain text.
|
|
181
|
+
function htmlToMarkdown(fragment, baseUrl) {
|
|
182
|
+
let out = fragment;
|
|
183
|
+
out = out.replace(/<a\b[^>]*\bhref\s*=\s*["']([^"']+)["'][^>]*>([\s\S]*?)<\/a>/gi, (_m, href, text) => {
|
|
184
|
+
const label = collapseWhitespace(decodeEntities(stripTags(text))).replace(/\n+/g, " ").trim();
|
|
185
|
+
const resolved = absoluteUrl(href, baseUrl);
|
|
186
|
+
if (!label) return "";
|
|
187
|
+
return resolved ? `[${label}](${resolved})` : label;
|
|
188
|
+
});
|
|
189
|
+
out = out.replace(/<\/(h[1-6])\s*>/gi, "\n\n");
|
|
190
|
+
out = out.replace(/<(h[1-6])\b[^>]*>/gi, (_m, tag) => `\n\n${"#".repeat(Number(tag[1]))} `);
|
|
191
|
+
out = out.replace(/<li\b[^>]*>/gi, "\n- ");
|
|
192
|
+
out = out.replace(/<\/(p|div|section|ul|ol|tr|table|blockquote)\s*>/gi, "\n\n");
|
|
193
|
+
out = out.replace(/<br\s*\/?>(?=)/gi, "\n");
|
|
194
|
+
out = stripTags(out);
|
|
195
|
+
out = decodeEntities(out);
|
|
196
|
+
// Drop empty heading markers left when a heading's text lived in nested blocks
|
|
197
|
+
// that were stripped (avoids junk lines like a bare "#" or "##").
|
|
198
|
+
out = out.replace(/^[ \t]*#{1,6}[ \t]*$/gm, "");
|
|
199
|
+
// Strip control/bidi characters BEFORE collapsing: if they sit between
|
|
200
|
+
// newlines they break up blank-line runs, and stripping them afterward would
|
|
201
|
+
// leave long orphaned newline runs that nothing re-collapses.
|
|
202
|
+
out = sanitizeText(out);
|
|
203
|
+
return collapseWhitespace(out);
|
|
204
|
+
}
|
|
205
|
+
|
|
206
|
+
// renderConcept re-adds the title as the top H1, so drop a leading body heading
|
|
207
|
+
// that merely repeats it (avoids "# Title" appearing twice). A *different* first
|
|
208
|
+
// heading is kept — it's real page structure.
|
|
209
|
+
function stripLeadingDuplicateH1(body, title) {
|
|
210
|
+
const m = /^#\s+(.+?)\s*\n+/.exec(body);
|
|
211
|
+
if (!m) return body;
|
|
212
|
+
const norm = (s) => String(s).replace(/\s+/g, " ").trim().toLowerCase();
|
|
213
|
+
return norm(m[1]) === norm(title) ? body.slice(m[0].length) : body;
|
|
214
|
+
}
|
|
215
|
+
|
|
216
|
+
export function absoluteUrl(href, baseUrl) {
|
|
217
|
+
const raw = (href || "").trim();
|
|
218
|
+
if (!raw || raw.startsWith("#") || /^(javascript|mailto|tel):/i.test(raw)) return "";
|
|
219
|
+
try {
|
|
220
|
+
return new URL(raw, baseUrl).toString();
|
|
221
|
+
} catch {
|
|
222
|
+
return "";
|
|
223
|
+
}
|
|
224
|
+
}
|
|
225
|
+
|
|
226
|
+
// Collect same-host outbound links (for future cross-linking / phase 2).
|
|
227
|
+
export function collectLinks(fragment, baseUrl) {
|
|
228
|
+
const links = [];
|
|
229
|
+
const seen = new Set();
|
|
230
|
+
let host = "";
|
|
231
|
+
try {
|
|
232
|
+
host = new URL(baseUrl).host;
|
|
233
|
+
} catch {
|
|
234
|
+
return links;
|
|
235
|
+
}
|
|
236
|
+
const re = /<a\b[^>]*\bhref\s*=\s*["']([^"']+)["'][^>]*>([\s\S]*?)<\/a>/gi;
|
|
237
|
+
let m;
|
|
238
|
+
while ((m = re.exec(fragment))) {
|
|
239
|
+
const resolved = absoluteUrl(m[1], baseUrl);
|
|
240
|
+
if (!resolved) continue;
|
|
241
|
+
let u;
|
|
242
|
+
try {
|
|
243
|
+
u = new URL(resolved);
|
|
244
|
+
} catch {
|
|
245
|
+
continue;
|
|
246
|
+
}
|
|
247
|
+
if (u.host !== host) continue;
|
|
248
|
+
const key = u.origin + u.pathname;
|
|
249
|
+
if (seen.has(key)) continue;
|
|
250
|
+
seen.add(key);
|
|
251
|
+
const text = collapseWhitespace(decodeEntities(stripTags(m[2]))).replace(/\n+/g, " ").trim();
|
|
252
|
+
links.push({ url: u.toString(), path: u.pathname, text });
|
|
253
|
+
}
|
|
254
|
+
return links;
|
|
255
|
+
}
|
|
256
|
+
|
|
257
|
+
// url path -> concept slug/filename. Root -> "home". Avoids reserved names.
|
|
258
|
+
export function slugForUrl(url) {
|
|
259
|
+
let pathname = "/";
|
|
260
|
+
try {
|
|
261
|
+
pathname = new URL(url).pathname || "/";
|
|
262
|
+
} catch {
|
|
263
|
+
pathname = "/";
|
|
264
|
+
}
|
|
265
|
+
let slug = pathname
|
|
266
|
+
.replace(/\.(html?|php|aspx?)$/i, "")
|
|
267
|
+
.replace(/^\/+|\/+$/g, "")
|
|
268
|
+
.replace(/[^a-zA-Z0-9]+/g, "-")
|
|
269
|
+
.replace(/^-+|-+$/g, "")
|
|
270
|
+
.toLowerCase();
|
|
271
|
+
if (!slug) slug = "home";
|
|
272
|
+
if (RESERVED_FILENAMES.has(`${slug}.md`)) slug = `${slug}-page`;
|
|
273
|
+
return slug;
|
|
274
|
+
}
|
|
275
|
+
|
|
276
|
+
function humanizeFromUrl(url) {
|
|
277
|
+
try {
|
|
278
|
+
const u = new URL(url);
|
|
279
|
+
const seg = (u.pathname.replace(/^\/+|\/+$/g, "").split("/").pop() || u.hostname)
|
|
280
|
+
.replace(/\.(html?|php|aspx?)$/i, "")
|
|
281
|
+
.replace(/[-_]+/g, " ")
|
|
282
|
+
.trim();
|
|
283
|
+
return seg ? seg.replace(/\b\w/g, (c) => c.toUpperCase()) : u.hostname;
|
|
284
|
+
} catch {
|
|
285
|
+
return url;
|
|
286
|
+
}
|
|
287
|
+
}
|
|
288
|
+
|
|
289
|
+
// Extract one page into an OKF concept (pure). No timestamp — caller adds it.
|
|
290
|
+
export function extractConcept(html, url) {
|
|
291
|
+
const source = String(html || "");
|
|
292
|
+
const title =
|
|
293
|
+
firstMatch(source, /<title\b[^>]*>([\s\S]*?)<\/title>/i) ||
|
|
294
|
+
firstMatch(source, /<h1\b[^>]*>([\s\S]*?)<\/h1>/i) ||
|
|
295
|
+
humanizeFromUrl(url);
|
|
296
|
+
const description = metaContent(source, "description") || metaContent(source, "og:description");
|
|
297
|
+
const resource = absoluteUrl(canonicalUrl(source), url) || url;
|
|
298
|
+
const region = mainRegion(removeNoise(source));
|
|
299
|
+
const body = stripLeadingDuplicateH1(htmlToMarkdown(region, url), title);
|
|
300
|
+
const links = collectLinks(region, url);
|
|
301
|
+
let firstSentence = description;
|
|
302
|
+
if (!firstSentence && body) {
|
|
303
|
+
const firstPara = body.split("\n").find((line) => line && !line.startsWith("#"));
|
|
304
|
+
if (firstPara) firstSentence = firstPara.split(/(?<=[.!?])\s/)[0].slice(0, 300);
|
|
305
|
+
}
|
|
306
|
+
return {
|
|
307
|
+
type: "WebPage",
|
|
308
|
+
title: sanitizeText(title).slice(0, 300),
|
|
309
|
+
description: sanitizeText(firstSentence || "").slice(0, 300),
|
|
310
|
+
resource,
|
|
311
|
+
body, // already sanitized-then-collapsed inside htmlToMarkdown
|
|
312
|
+
links: links.map((link) => ({ ...link, text: sanitizeText(link.text) })),
|
|
313
|
+
};
|
|
314
|
+
}
|
|
315
|
+
|
|
316
|
+
function yamlScalar(value) {
|
|
317
|
+
const s = sanitizeText(String(value == null ? "" : value))
|
|
318
|
+
.replace(/\\/g, "\\\\")
|
|
319
|
+
.replace(/"/g, '\\"')
|
|
320
|
+
.replace(/[\r\n]+/g, " ");
|
|
321
|
+
return `"${s}"`;
|
|
322
|
+
}
|
|
323
|
+
|
|
324
|
+
export function renderConcept({ type, title, description, resource, tags, timestamp, body, seeAlso }) {
|
|
325
|
+
const lines = ["---", `type: ${yamlScalar(type || "WebPage")}`];
|
|
326
|
+
if (title) lines.push(`title: ${yamlScalar(title)}`);
|
|
327
|
+
if (description) lines.push(`description: ${yamlScalar(description)}`);
|
|
328
|
+
// Quoted: a hostile <link rel=canonical> could otherwise smuggle "#" or ": "
|
|
329
|
+
// into the plain scalar and corrupt or truncate the frontmatter.
|
|
330
|
+
if (resource) lines.push(`resource: ${yamlScalar(resource)}`);
|
|
331
|
+
if (Array.isArray(tags) && tags.length) lines.push(`tags: [${tags.map(yamlScalar).join(", ")}]`);
|
|
332
|
+
if (timestamp) lines.push(`timestamp: ${timestamp}`);
|
|
333
|
+
lines.push("---", "");
|
|
334
|
+
if (title) lines.push(`# ${title}`, "");
|
|
335
|
+
lines.push(body || "");
|
|
336
|
+
if (Array.isArray(seeAlso) && seeAlso.length) {
|
|
337
|
+
lines.push("", "## See also");
|
|
338
|
+
for (const item of seeAlso) lines.push(`- [${item.title}](/${item.slug}.md)`);
|
|
339
|
+
}
|
|
340
|
+
return `${lines.join("\n").trimEnd()}\n`;
|
|
341
|
+
}
|
|
342
|
+
|
|
343
|
+
export function renderIndex(concepts, { title } = {}) {
|
|
344
|
+
const lines = ["---", `okf_version: "${OKF_VERSION}"`, "---", "", `# ${title || "Knowledge Bundle"}`, ""];
|
|
345
|
+
for (const c of concepts) {
|
|
346
|
+
const desc = c.description ? ` - ${c.description}` : "";
|
|
347
|
+
lines.push(`* [${c.title || c.slug}](/${c.slug}.md)${desc}`);
|
|
348
|
+
}
|
|
349
|
+
return `${lines.join("\n").trimEnd()}\n`;
|
|
350
|
+
}
|
|
351
|
+
|
|
352
|
+
export function renderLog({ url, timestamp, cliVersion, conceptCount, notes }) {
|
|
353
|
+
const lines = [
|
|
354
|
+
"# Generation log",
|
|
355
|
+
"",
|
|
356
|
+
`- generator: @sleepwalkerai/cli okf export`,
|
|
357
|
+
`- cli_version: ${cliVersion || "unknown"}`,
|
|
358
|
+
`- extractor: local (dependency-free)`,
|
|
359
|
+
`- source_url: ${url}`,
|
|
360
|
+
`- generated_at: ${timestamp}`,
|
|
361
|
+
`- concepts: ${conceptCount}`,
|
|
362
|
+
`- okf_version: ${OKF_VERSION}`,
|
|
363
|
+
];
|
|
364
|
+
for (const note of notes || []) lines.push(`- note: ${note}`);
|
|
365
|
+
return `${lines.join("\n")}\n`;
|
|
366
|
+
}
|
|
367
|
+
|
|
368
|
+
// Assemble a full single-page bundle (pure). Returns { files, summary }.
|
|
369
|
+
export function buildBundle({ url, html, now, cliVersion, extraNotes }) {
|
|
370
|
+
const timestamp = now || new Date().toISOString();
|
|
371
|
+
const concept = extractConcept(html, url);
|
|
372
|
+
const slug = slugForUrl(concept.resource || url);
|
|
373
|
+
const notes = Array.isArray(extraNotes) ? extraNotes.filter(Boolean).map(sanitizeText) : [];
|
|
374
|
+
if (!concept.body) notes.push("no readable content extracted from the page");
|
|
375
|
+
|
|
376
|
+
const files = {};
|
|
377
|
+
files[`${slug}.md`] = renderConcept({ ...concept, timestamp });
|
|
378
|
+
files["index.md"] = renderIndex([{ slug, title: concept.title, description: concept.description }], {
|
|
379
|
+
title: concept.title,
|
|
380
|
+
});
|
|
381
|
+
files["log.md"] = renderLog({ url, timestamp, cliVersion, conceptCount: 1, notes });
|
|
382
|
+
|
|
383
|
+
return {
|
|
384
|
+
files,
|
|
385
|
+
summary: {
|
|
386
|
+
url,
|
|
387
|
+
resource: concept.resource,
|
|
388
|
+
conceptCount: 1,
|
|
389
|
+
files: Object.keys(files),
|
|
390
|
+
credits: 0,
|
|
391
|
+
title: concept.title,
|
|
392
|
+
notes,
|
|
393
|
+
},
|
|
394
|
+
};
|
|
395
|
+
}
|
|
396
|
+
|
|
397
|
+
export function defaultOutDir(url) {
|
|
398
|
+
let host = "okf";
|
|
399
|
+
try {
|
|
400
|
+
host = new URL(url).hostname.replace(/^www\./, "") || "okf";
|
|
401
|
+
} catch {
|
|
402
|
+
host = "okf";
|
|
403
|
+
}
|
|
404
|
+
return `./${host}-okf`;
|
|
405
|
+
}
|
|
406
|
+
|
|
407
|
+
// The one I/O function; kept out of the pure path so tests don't need disk.
|
|
408
|
+
export async function writeBundle(outDir, files, { force = false } = {}) {
|
|
409
|
+
const dir = path.resolve(outDir);
|
|
410
|
+
let existing = [];
|
|
411
|
+
try {
|
|
412
|
+
existing = await readdir(dir);
|
|
413
|
+
} catch {
|
|
414
|
+
existing = [];
|
|
415
|
+
}
|
|
416
|
+
if (existing.length && !force) {
|
|
417
|
+
const err = new Error(`Output directory ${outDir} is not empty. Use --force to overwrite.`);
|
|
418
|
+
err.exitCode = 1;
|
|
419
|
+
throw err;
|
|
420
|
+
}
|
|
421
|
+
await mkdir(dir, { recursive: true });
|
|
422
|
+
const written = [];
|
|
423
|
+
for (const [name, content] of Object.entries(files)) {
|
|
424
|
+
const target = path.join(dir, name);
|
|
425
|
+
await mkdir(path.dirname(target), { recursive: true });
|
|
426
|
+
await writeFile(target, content, "utf8");
|
|
427
|
+
written.push(name);
|
|
428
|
+
}
|
|
429
|
+
return { dir, written };
|
|
430
|
+
}
|
package/src/theme.js
CHANGED
|
@@ -151,6 +151,9 @@ ${section("Read")}
|
|
|
151
151
|
${command("sleepwalker tests list")} ${muted("[--limit 20] [--type ai_citations|content_intelligence]")}
|
|
152
152
|
${command("sleepwalker reports by-url <url>")} ${muted("[--type ai_citations|content_intelligence]")}
|
|
153
153
|
|
|
154
|
+
${section("Local (free — no account, no credits)")}
|
|
155
|
+
${command("sleepwalker okf export <url>")} ${muted("[--out <dir>] [--force] build an Open Knowledge Format bundle on your machine")}
|
|
156
|
+
|
|
154
157
|
${section("Actions")}
|
|
155
158
|
${command("sleepwalker page serialize <url>")} ${muted("[--max-chars 4000] [--offset 0]")}
|
|
156
159
|
${command("sleepwalker visibility suggest-prompts <url>")} ${muted("--brand <brand>")}
|