@openbooknetwork/kitap 0.3.0 → 0.3.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +24 -0
- package/package.json +3 -2
- package/scripts/cloudflare-book.mjs +115 -0
- package/scripts/cloudflare-redirect.mjs +62 -0
- package/src/candidates.mjs +112 -0
- package/src/cli.mjs +4 -3
- package/src/index.mjs +1 -0
package/README.md
CHANGED
|
@@ -50,6 +50,7 @@ kitap check <story> [page id ...] [--places]
|
|
|
50
50
|
kitap dev [--story <id>] [--out <dir>] [--port <n>]
|
|
51
51
|
kitap i18n <lang> [--story <id>]
|
|
52
52
|
kitap image <file or url> <name> --caption <text> [--credit <text>] [--source <url>] [--story <id>]
|
|
53
|
+
kitap image-candidates <story> --queries <file> [--limit <n>] [--pages <id,id>] [--out <name>]
|
|
53
54
|
kitap rehash | kitap upload [--story <id>] [--dry-run] | kitap deploy [--dry-run]
|
|
54
55
|
```
|
|
55
56
|
|
|
@@ -57,6 +58,11 @@ An engine's own binary runs the same commands with its engine, `harita build` fo
|
|
|
57
58
|
`--strict` fails on a missing translation or an object the store lacks; `--local-assets` shows the files on this
|
|
58
59
|
machine before they are uploaded.
|
|
59
60
|
|
|
61
|
+
`image-candidates` searches Wikimedia Commons for each page of a story and downloads what it finds to
|
|
62
|
+
`<story>/.candidates/<page>/`, listed with author, license and source in `.candidates/index.yaml`. The queries file
|
|
63
|
+
is keyed by story, then page: `{ queries: [search terms], articles: [Wikipedia titles or URLs], files: [Commons
|
|
64
|
+
files] }`. A file the book holds already is listed with `held: <id>`. `kitap image <url>` adds a chosen one.
|
|
65
|
+
|
|
60
66
|
## An engine
|
|
61
67
|
|
|
62
68
|
An engine is a package whose default export is a module with an id and hooks. Kitap resolves `engine: <id>` of
|
|
@@ -97,6 +103,24 @@ and, as a module, dispatches `kitap-engine` once set. `host` carries the bundle
|
|
|
97
103
|
|
|
98
104
|
The test engine in `test/engines/plain.mjs` shows the least an engine needs.
|
|
99
105
|
|
|
106
|
+
## Deploy
|
|
107
|
+
|
|
108
|
+
A book serves from Cloudflare: the images and the engine's objects from an R2 bucket behind `assets.<zone>`, the
|
|
109
|
+
pages from a second bucket through a Worker on the zone. `site.yaml` names them in `url`, `assets` and `pages`, and
|
|
110
|
+
one script sets them up from a content project:
|
|
111
|
+
|
|
112
|
+
```
|
|
113
|
+
CF_TOKEN=<token> node node_modules/@openbooknetwork/kitap/scripts/cloudflare-book.mjs [--lock]
|
|
114
|
+
```
|
|
115
|
+
|
|
116
|
+
It adds the zone and prints its nameservers for the registrar, creates both buckets, attaches the assets domain,
|
|
117
|
+
sets CORS, the cache rule, the SVG header rule and tiered cache, and with `--lock` the bucket lock on `images/`.
|
|
118
|
+
Run it again once the zone is active. Then three tokens by hand: an R2 token on the assets bucket for `kitap
|
|
119
|
+
upload`, one on the site bucket as the `AWS_ACCESS_KEY_ID` and `AWS_SECRET_ACCESS_KEY` secrets of the deploy
|
|
120
|
+
workflow, and a wrangler token with Workers Scripts Edit and Workers Routes Edit as `CLOUDFLARE_API_TOKEN` with
|
|
121
|
+
the account id as `CLOUDFLARE_ACCOUNT_ID`. `scripts/cloudflare-redirect.mjs <old zone> <new zone>` sends a
|
|
122
|
+
retired domain to the new one.
|
|
123
|
+
|
|
100
124
|
## Work on it
|
|
101
125
|
|
|
102
126
|
```
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@openbooknetwork/kitap",
|
|
3
|
-
"version": "0.3.
|
|
3
|
+
"version": "0.3.1",
|
|
4
4
|
"description": "The book under the Open Book Network sites: stories of pages with a text beside a picture an engine draws, built to a static site in every language",
|
|
5
5
|
"keywords": [
|
|
6
6
|
"static-site",
|
|
@@ -31,7 +31,8 @@
|
|
|
31
31
|
},
|
|
32
32
|
"files": [
|
|
33
33
|
"bin",
|
|
34
|
-
"src"
|
|
34
|
+
"src",
|
|
35
|
+
"scripts"
|
|
35
36
|
],
|
|
36
37
|
"publishConfig": {
|
|
37
38
|
"access": "public"
|
|
@@ -0,0 +1,115 @@
|
|
|
1
|
+
#!/usr/bin/env node
|
|
2
|
+
// Sets a book up on Cloudflare from its site.yaml: the zone of `url`, the assets bucket with its custom domain and
|
|
3
|
+
// CORS, the site bucket, the cache rule and the SVG header rule on the assets host, and tiered cache. Every step
|
|
4
|
+
// is skipped when it is already done, so the script can run again after the zone goes active.
|
|
5
|
+
//
|
|
6
|
+
// CF_TOKEN=<token> node node_modules/@openbooknetwork/kitap/scripts/cloudflare-book.mjs [--lock] [--dry-run]
|
|
7
|
+
//
|
|
8
|
+
// The token needs, on the account: Zone Edit, Workers R2 Storage Edit, DNS Edit; on the zone: Zone Read, Zone
|
|
9
|
+
// Settings Edit, Cache Rules Edit, Transform Rules Edit. --lock adds the bucket lock on images/, for after the
|
|
10
|
+
// first upload is checked. What stays manual: the nameservers at the registrar, and the R2 and wrangler tokens.
|
|
11
|
+
import fs from 'node:fs';
|
|
12
|
+
import { parseArgs } from 'node:util';
|
|
13
|
+
import yaml from 'js-yaml';
|
|
14
|
+
|
|
15
|
+
const API = 'https://api.cloudflare.com/client/v4';
|
|
16
|
+
const CSP = "default-src 'none'; style-src 'unsafe-inline'; img-src data:";
|
|
17
|
+
|
|
18
|
+
const { values } = parseArgs({ options: { lock: { type: 'boolean' }, 'dry-run': { type: 'boolean' }, account: { type: 'string' } } });
|
|
19
|
+
const site = yaml.load(fs.readFileSync('site.yaml', 'utf8'));
|
|
20
|
+
for (const k of ['url', 'assets', 'pages']) if (!site[k]) fail(`site.yaml has no ${k}, see kitap's README under Deploy`);
|
|
21
|
+
const zoneName = new URL(site.url).hostname.replace(/^www\./, '');
|
|
22
|
+
const assetsHost = new URL(site.assets.url).hostname;
|
|
23
|
+
const assetsBucket = site.assets.bucket, siteBucket = site.pages.bucket;
|
|
24
|
+
const token = process.env.CF_TOKEN, dry = values['dry-run'];
|
|
25
|
+
if (!token && !dry) fail('set CF_TOKEN to a Cloudflare API token, see the head of this script for its permissions');
|
|
26
|
+
|
|
27
|
+
console.log(`${zoneName}: assets ${assetsBucket} at ${assetsHost}, site ${siteBucket}`);
|
|
28
|
+
if (dry) { console.log('dry run: nothing is sent'); process.exit(0); }
|
|
29
|
+
|
|
30
|
+
const account = values.account ?? await accountId();
|
|
31
|
+
const zone = await ensureZone();
|
|
32
|
+
await ensureBucket(assetsBucket);
|
|
33
|
+
await ensureBucket(siteBucket);
|
|
34
|
+
await cors(assetsBucket);
|
|
35
|
+
await managedDomainOff(assetsBucket);
|
|
36
|
+
await customDomain(assetsBucket);
|
|
37
|
+
await rules();
|
|
38
|
+
if (values.lock) await lock(assetsBucket);
|
|
39
|
+
console.log(zone.status === 'active' ? `${zoneName} is set up` : `${zoneName} is ${zone.status}: point the registrar at ${zone.name_servers.join(' and ')}, then run this again`);
|
|
40
|
+
|
|
41
|
+
async function accountId() {
|
|
42
|
+
const list = await call('GET', '/accounts');
|
|
43
|
+
if (list.length !== 1) fail(`the token sees ${list.length} accounts, pass --account <id>`);
|
|
44
|
+
return list[0].id;
|
|
45
|
+
}
|
|
46
|
+
|
|
47
|
+
async function ensureZone() {
|
|
48
|
+
const found = await call('GET', `/zones?name=${zoneName}`);
|
|
49
|
+
if (found.length) { console.log(`zone ${zoneName}: ${found[0].status}`); return found[0]; }
|
|
50
|
+
const z = await call('POST', '/zones', { name: zoneName, account: { id: account }, type: 'full' });
|
|
51
|
+
console.log(`zone ${zoneName}: added, nameservers ${z.name_servers.join(' and ')}`);
|
|
52
|
+
return z;
|
|
53
|
+
}
|
|
54
|
+
|
|
55
|
+
async function ensureBucket(name) {
|
|
56
|
+
const res = await fetch(`${API}/accounts/${account}/r2/buckets/${name}`, { headers: { Authorization: `Bearer ${token}` } });
|
|
57
|
+
if (res.ok) { console.log(`bucket ${name}: exists`); return; }
|
|
58
|
+
await call('POST', `/accounts/${account}/r2/buckets`, { name });
|
|
59
|
+
console.log(`bucket ${name}: created`);
|
|
60
|
+
}
|
|
61
|
+
|
|
62
|
+
async function cors(bucket) {
|
|
63
|
+
await call('PUT', `/accounts/${account}/r2/buckets/${bucket}/cors`, { rules: [{ allowed: { origins: ['*'], methods: ['GET', 'HEAD'] } }] });
|
|
64
|
+
console.log(`bucket ${bucket}: CORS GET and HEAD from anywhere`);
|
|
65
|
+
}
|
|
66
|
+
|
|
67
|
+
// the r2.dev URL skips the cache, so the custom domain is the only way in
|
|
68
|
+
async function managedDomainOff(bucket) {
|
|
69
|
+
await call('PUT', `/accounts/${account}/r2/buckets/${bucket}/domains/managed`, { enabled: false });
|
|
70
|
+
}
|
|
71
|
+
|
|
72
|
+
async function customDomain(bucket) {
|
|
73
|
+
const { domains } = await call('GET', `/accounts/${account}/r2/buckets/${bucket}/domains/custom`);
|
|
74
|
+
if (domains.some(d => d.domain === assetsHost)) { console.log(`bucket ${bucket}: ${assetsHost} attached`); return; }
|
|
75
|
+
if (zone.status !== 'active') { console.log(`bucket ${bucket}: ${assetsHost} waits for the zone to go active`); return; }
|
|
76
|
+
await call('POST', `/accounts/${account}/r2/buckets/${bucket}/domains/custom`, { domain: assetsHost, zoneId: zone.id, enabled: true, minTLS: '1.2' });
|
|
77
|
+
console.log(`bucket ${bucket}: ${assetsHost} attached`);
|
|
78
|
+
}
|
|
79
|
+
|
|
80
|
+
async function rules() {
|
|
81
|
+
await call('PUT', `/zones/${zone.id}/rulesets/phases/http_request_cache_settings/entrypoint`, { rules: [{
|
|
82
|
+
description: 'assets',
|
|
83
|
+
expression: `(http.host eq "${assetsHost}")`,
|
|
84
|
+
action: 'set_cache_settings',
|
|
85
|
+
action_parameters: {
|
|
86
|
+
cache: true,
|
|
87
|
+
cache_key: { custom_key: { query_string: { exclude: { all: true } } } },
|
|
88
|
+
// a 404 stays uncached, so a strict build right after an upload sees the new keys
|
|
89
|
+
edge_ttl: { mode: 'respect_origin', status_code_ttl: [{ status_code: 404, value: -1 }] },
|
|
90
|
+
},
|
|
91
|
+
}] });
|
|
92
|
+
await call('PUT', `/zones/${zone.id}/rulesets/phases/http_response_headers_transform/entrypoint`, { rules: [{
|
|
93
|
+
description: 'svg csp',
|
|
94
|
+
expression: `(http.host eq "${assetsHost}" and ends_with(http.request.uri.path, ".svg"))`,
|
|
95
|
+
action: 'rewrite',
|
|
96
|
+
action_parameters: { headers: { 'Content-Security-Policy': { operation: 'set', value: CSP } } },
|
|
97
|
+
}] });
|
|
98
|
+
await call('PATCH', `/zones/${zone.id}/cache/tiered_cache_smart_topology_enable`, { value: 'on' });
|
|
99
|
+
console.log(`zone ${zoneName}: cache rule, SVG header rule and tiered cache`);
|
|
100
|
+
}
|
|
101
|
+
|
|
102
|
+
// the store then refuses to delete or overwrite an image, whoever asks
|
|
103
|
+
async function lock(bucket) {
|
|
104
|
+
await call('PUT', `/accounts/${account}/r2/buckets/${bucket}/lock`, { rules: [{ id: 'images', enabled: true, prefix: 'images/', condition: { type: 'Indefinite' } }] });
|
|
105
|
+
console.log(`bucket ${bucket}: images/ locked`);
|
|
106
|
+
}
|
|
107
|
+
|
|
108
|
+
async function call(method, path, body) {
|
|
109
|
+
const res = await fetch(API + path, { method, body: body && JSON.stringify(body), headers: { Authorization: `Bearer ${token}`, 'Content-Type': 'application/json' } });
|
|
110
|
+
const out = await res.json();
|
|
111
|
+
if (!out.success) fail(`${method} ${path} failed: ${JSON.stringify(out.errors, null, 2)}`);
|
|
112
|
+
return out.result;
|
|
113
|
+
}
|
|
114
|
+
|
|
115
|
+
function fail(message) { console.error(message); process.exit(1); }
|
|
@@ -0,0 +1,62 @@
|
|
|
1
|
+
#!/usr/bin/env node
|
|
2
|
+
// Sends an old zone to a new one by 301, the apex and the assets host alike, path and query kept:
|
|
3
|
+
//
|
|
4
|
+
// CF_TOKEN=<token> node node_modules/@openbooknetwork/kitap/scripts/cloudflare-redirect.mjs openhistoryatlas.org openhistorybook.org [--dry-run]
|
|
5
|
+
//
|
|
6
|
+
// The token needs Zone Read and Single Redirect Edit on the old zone. The rule replaces the zone's own redirect
|
|
7
|
+
// rules, so run it on a zone that has none or that this script made.
|
|
8
|
+
import { parseArgs } from 'node:util';
|
|
9
|
+
|
|
10
|
+
const API = 'https://api.cloudflare.com/client/v4';
|
|
11
|
+
const { values, positionals } = parseArgs({ allowPositionals: true, options: { 'dry-run': { type: 'boolean' } } });
|
|
12
|
+
const [from, to] = positionals;
|
|
13
|
+
if (!from || !to) fail('usage: CF_TOKEN=<token> node cloudflare-redirect.mjs <old zone> <new zone> [--dry-run]');
|
|
14
|
+
const token = process.env.CF_TOKEN;
|
|
15
|
+
if (!token && !values['dry-run']) fail('set CF_TOKEN to a Cloudflare API token with Zone Read and Single Redirect Edit');
|
|
16
|
+
|
|
17
|
+
const body = redirectRules(from, to);
|
|
18
|
+
if (values['dry-run']) console.log(JSON.stringify(body, null, 2));
|
|
19
|
+
else {
|
|
20
|
+
await call('PUT', `/zones/${await zoneId(from)}/rulesets/phases/http_request_dynamic_redirect/entrypoint`, body);
|
|
21
|
+
console.log(`${from} redirects to ${to}`);
|
|
22
|
+
}
|
|
23
|
+
|
|
24
|
+
// One rule per hostname: the apex to the new apex, the assets host to the new assets host, path and query kept.
|
|
25
|
+
function redirectRules(from, to) {
|
|
26
|
+
const rule = (fromHost, toHost) => ({
|
|
27
|
+
description: `${fromHost} to ${toHost}`,
|
|
28
|
+
expression: `(http.host eq "${fromHost}")`,
|
|
29
|
+
action: 'redirect',
|
|
30
|
+
action_parameters: { from_value: {
|
|
31
|
+
status_code: 301,
|
|
32
|
+
target_url: { expression: `concat("https://${toHost}", http.request.uri.path)` },
|
|
33
|
+
preserve_query_string: true,
|
|
34
|
+
} },
|
|
35
|
+
});
|
|
36
|
+
return { rules: [rule(from, to), rule(`assets.${from}`, `assets.${to}`)] };
|
|
37
|
+
}
|
|
38
|
+
|
|
39
|
+
async function zoneId(name) {
|
|
40
|
+
const res = await fetch(`${API}/zones?name=${name}`, { headers: { Authorization: `Bearer ${token}` } });
|
|
41
|
+
const out = await res.json();
|
|
42
|
+
if (!out.success) fail(`zone lookup failed: ${JSON.stringify(out.errors, null, 2)}`);
|
|
43
|
+
if (!out.result.length) {
|
|
44
|
+
fail(`the token sees no zone named ${name}: give it Zone Read on that zone`);
|
|
45
|
+
}
|
|
46
|
+
return out.result[0].id;
|
|
47
|
+
}
|
|
48
|
+
|
|
49
|
+
async function call(method, path, body) {
|
|
50
|
+
const res = await fetch(API + path, {
|
|
51
|
+
method, body: JSON.stringify(body),
|
|
52
|
+
headers: { Authorization: `Bearer ${token}`, 'Content-Type': 'application/json' },
|
|
53
|
+
});
|
|
54
|
+
const out = await res.json();
|
|
55
|
+
if (!out.success) fail(`${method} ${path} failed: ${JSON.stringify(out.errors, null, 2)}`);
|
|
56
|
+
return out.result;
|
|
57
|
+
}
|
|
58
|
+
|
|
59
|
+
function fail(message) {
|
|
60
|
+
console.error(message);
|
|
61
|
+
process.exit(1);
|
|
62
|
+
}
|
|
@@ -0,0 +1,112 @@
|
|
|
1
|
+
// kitap image-candidates: searches Wikimedia Commons for each page's queries and articles, and downloads what it finds
|
|
2
|
+
// to <story folder>/.candidates/<page>/ for review, listed with author, license and source in .candidates/index.yaml.
|
|
3
|
+
// kitap image then adds a chosen one by its url. A file the book holds already, by its Commons page or its bytes, is
|
|
4
|
+
// listed with held: <id> and not kept.
|
|
5
|
+
import fs from 'fs';
|
|
6
|
+
import path from 'path';
|
|
7
|
+
import yaml from 'js-yaml';
|
|
8
|
+
import { storyFolders } from './folders.mjs';
|
|
9
|
+
import { imageIndex, findImage } from './image.mjs';
|
|
10
|
+
|
|
11
|
+
const API = 'https://commons.wikimedia.org/w/api.php';
|
|
12
|
+
const BITMAP = /\.(jpe?g|png|tiff?)$/i;
|
|
13
|
+
// an article is a Wikipedia URL or an English Wikipedia title
|
|
14
|
+
const articleApi = a => { const m = a.match(/^https?:\/\/([a-z-]+)\.wikipedia\.org\/wiki\/([^?#]+)/); return m ? [`https://${m[1]}.wikipedia.org/w/api.php`, decodeURIComponent(m[2]).replace(/_/g, ' ')] : ['https://en.wikipedia.org/w/api.php', a]; };
|
|
15
|
+
const strip = s => String(s ?? '').replace(/<[^>]+>/g, '').replace(/\s+/g, ' ').trim();
|
|
16
|
+
|
|
17
|
+
// queries is a YAML file keyed by story id, then by page id: { queries: [search terms], articles: [titles or URLs],
|
|
18
|
+
// files: [Commons file names] }, or a bare list of search terms. pages refetches those pages, out writes into
|
|
19
|
+
// .candidates/<out>/ so agents fetching at the same time keep separate index files. pause scales the waits.
|
|
20
|
+
export async function imageCandidates({ root = process.cwd(), story, queries, limit = 6, pages = null, out = '', log = console.log, fetch = globalThis.fetch, pause = 1 } = {}) {
|
|
21
|
+
const stories = storyFolders(path.join(root, 'content'), p => path.relative(root, p)); // full paths by folder name
|
|
22
|
+
if (!stories.has(story)) throw new Error(`"${story}" is not a story folder under content/`);
|
|
23
|
+
if (!queries) throw new Error('name the queries file with --queries <file>: YAML keyed by story, then page, each with queries, articles or files');
|
|
24
|
+
const spec = yaml.load(fs.readFileSync(path.resolve(root, queries), 'utf8'))?.[story];
|
|
25
|
+
if (!spec) throw new Error(`${queries}: no queries for the story "${story}"`);
|
|
26
|
+
const outDir = path.join(stories.get(story), '.candidates', out);
|
|
27
|
+
fs.mkdirSync(outDir, { recursive: true });
|
|
28
|
+
|
|
29
|
+
// Wikimedia asks for a contact in the user agent and throttles unnamed clients harder
|
|
30
|
+
const siteFile = path.join(root, 'site.yaml'), site = fs.existsSync(siteFile) ? yaml.load(fs.readFileSync(siteFile, 'utf8')) ?? {} : {};
|
|
31
|
+
const title = typeof site.title === 'object' ? Object.values(site.title)[0] : site.title;
|
|
32
|
+
const headers = { 'User-Agent': `${title ?? 'kitap'} image-candidates (${site.repository ?? site.url ?? 'https://github.com/openbooknetwork/kitap'})` };
|
|
33
|
+
const sleep = ms => new Promise(r => setTimeout(r, ms * pause));
|
|
34
|
+
// Commons answers 429 when asked too fast: wait as told, or doubling from two seconds, up to six tries
|
|
35
|
+
async function get(url) {
|
|
36
|
+
for (let attempt = 0; ; attempt++) {
|
|
37
|
+
// a dropped connection is retried like a 5xx
|
|
38
|
+
const res = await fetch(url, { headers }).catch(e => { if (attempt >= 6) throw e; return { ok: false, status: 599, headers: new Headers() }; });
|
|
39
|
+
if (res.ok) return res;
|
|
40
|
+
if ((res.status === 429 || res.status >= 500) && attempt < 6) {
|
|
41
|
+
const wait = Number(res.headers.get('retry-after')) * 1000 || 2000 * 2 ** attempt;
|
|
42
|
+
log(` ${res.status}, waiting ${wait / 1000}s`); await sleep(wait); continue;
|
|
43
|
+
}
|
|
44
|
+
throw new Error(`${res.status} for ${url}`);
|
|
45
|
+
}
|
|
46
|
+
}
|
|
47
|
+
const api = async (params, base = API) => (await get(base + '?' + new URLSearchParams({ format: 'json', ...params }))).json();
|
|
48
|
+
|
|
49
|
+
// resume: pages already in index.yaml are kept and skipped
|
|
50
|
+
const indexFile = path.join(outDir, 'index.yaml');
|
|
51
|
+
const index = fs.existsSync(indexFile) ? yaml.load(fs.readFileSync(indexFile, 'utf8')) ?? {} : {};
|
|
52
|
+
const save = () => fs.writeFileSync(indexFile, yaml.dump(index, { lineWidth: 120 }));
|
|
53
|
+
const held = imageIndex({ root });
|
|
54
|
+
const seen = new Set(Object.values(index).flat().map(c => 'File:' + c.title));
|
|
55
|
+
for (const [page, s] of Object.entries(spec)) {
|
|
56
|
+
if (pages && !pages.includes(page)) continue;
|
|
57
|
+
if (pages && index[page]) { for (const c of index[page]) seen.delete('File:' + c.title); delete index[page]; fs.rmSync(path.join(outDir, page), { recursive: true, force: true }); }
|
|
58
|
+
if (index[page]) { log(`${page}: done earlier, skipping`); continue; }
|
|
59
|
+
const terms = Array.isArray(s) ? s : s.queries ?? [], files = Array.isArray(s) ? [] : s.files ?? [], articles = Array.isArray(s) ? [] : s.articles ?? [];
|
|
60
|
+
const titles = files.map(f => f.startsWith('File:') ? f : 'File:' + f);
|
|
61
|
+
// every bitmap an article uses; files local to Wikipedia, usually non-free, have no Commons record and drop out below
|
|
62
|
+
for (const a of articles) {
|
|
63
|
+
const [base, t] = articleApi(a);
|
|
64
|
+
const r = await api({ action: 'query', prop: 'images', titles: t, imlimit: 'max', redirects: 1 }, base);
|
|
65
|
+
const found = Object.values(r.query?.pages ?? {});
|
|
66
|
+
if (!found.length || found.some(p => 'missing' in p)) log(` no article "${t}"`);
|
|
67
|
+
for (const p of found) for (const im of p.images ?? []) if (BITMAP.test(im.title)) titles.push(im.title);
|
|
68
|
+
await sleep(1000);
|
|
69
|
+
}
|
|
70
|
+
for (const q of terms) {
|
|
71
|
+
const r = await api({ action: 'query', list: 'search', srsearch: `${q} filetype:bitmap`, srnamespace: 6, srlimit: limit * 2 });
|
|
72
|
+
const hits = (r.query?.search ?? []).map(h => h.title).filter(t => BITMAP.test(t) && !seen.has(t) && !titles.includes(t));
|
|
73
|
+
titles.push(...hits.slice(0, limit));
|
|
74
|
+
await sleep(1000);
|
|
75
|
+
}
|
|
76
|
+
const picked = [...new Set(titles)].filter(t => !seen.has(t));
|
|
77
|
+
if (!picked.length) { log(`${page}: nothing found`); continue; }
|
|
78
|
+
const infoPages = [];
|
|
79
|
+
for (let i = 0; i < picked.length; i += 50) { // the API takes at most 50 titles per call
|
|
80
|
+
const info = await api({ action: 'query', prop: 'imageinfo', titles: picked.slice(i, i + 50).join('|'), iiprop: 'url|extmetadata|size', iiurlwidth: 1280 }); // a standard thumbnail step, Wikimedia throttles other widths
|
|
81
|
+
infoPages.push(...Object.values(info.query?.pages ?? {}));
|
|
82
|
+
}
|
|
83
|
+
const dir = path.join(outDir, page);
|
|
84
|
+
fs.mkdirSync(dir, { recursive: true });
|
|
85
|
+
index[page] = [];
|
|
86
|
+
let n = 0;
|
|
87
|
+
for (const p of infoPages) {
|
|
88
|
+
const ii = p.imageinfo?.[0]; if (!ii) continue;
|
|
89
|
+
const meta = ii.extmetadata ?? {};
|
|
90
|
+
// the extension comes from the downloaded thumbnail, so a long title cut to 80 characters keeps one
|
|
91
|
+
const url = ii.thumburl ?? ii.url, ext = path.extname(new URL(url).pathname).toLowerCase();
|
|
92
|
+
const base = p.title.replace(/^File:/, '').replace(/\.\w+$/, '').replace(/[^\w.-]+/g, '_').slice(0, 80);
|
|
93
|
+
const commonsPage = `https://commons.wikimedia.org/wiki/${encodeURIComponent(p.title)}`;
|
|
94
|
+
const entry = { title: p.title.replace(/^File:/, ''), page: commonsPage, url, author: strip(meta.Artist?.value), license: strip(meta.LicenseShortName?.value),
|
|
95
|
+
date: strip(meta.DateTimeOriginal?.value), description: strip(meta.ImageDescription?.value).slice(0, 300), width: ii.width, height: ii.height };
|
|
96
|
+
seen.add(p.title);
|
|
97
|
+
const known = findImage(held, { source: commonsPage });
|
|
98
|
+
if (known) { index[page].push({ file: null, held: known.id, ...entry }); log(` held already: ${known.id}`); continue; }
|
|
99
|
+
const name = `${String(++n).padStart(2, '0')}-${base}${ext}`, file = path.join(dir, name);
|
|
100
|
+
try { fs.writeFileSync(file, Buffer.from(await (await get(url)).arrayBuffer())); } catch (e) { log(` skip ${p.title}: ${e.message}`); continue; }
|
|
101
|
+
// the same bytes in an image folder that names no source: the download goes
|
|
102
|
+
const same = findImage(held, { file });
|
|
103
|
+
if (same) { fs.rmSync(file); index[page].push({ file: null, held: same.id, ...entry }); log(` held already: ${same.id}`); continue; }
|
|
104
|
+
index[page].push({ file: name, ...entry });
|
|
105
|
+
await sleep(800);
|
|
106
|
+
}
|
|
107
|
+
log(`${page}: ${index[page].length} candidates`);
|
|
108
|
+
save();
|
|
109
|
+
}
|
|
110
|
+
log(`wrote ${path.relative(root, indexFile)}`);
|
|
111
|
+
return index;
|
|
112
|
+
}
|
package/src/cli.mjs
CHANGED
|
@@ -1,16 +1,16 @@
|
|
|
1
1
|
// The command line of kitap and of every engine: the book's commands with the options they take. An engine's binary
|
|
2
2
|
// calls run with its engine and its own commands, which are tried first.
|
|
3
3
|
import { parseArgs } from 'util';
|
|
4
|
-
import { build, check, dev, i18n, image, rehash, upload, deploy } from './index.mjs';
|
|
4
|
+
import { build, check, dev, i18n, image, imageCandidates, rehash, upload, deploy } from './index.mjs';
|
|
5
5
|
|
|
6
|
-
export const USAGE = (name = 'kitap', extra = '') => `usage: ${name} build [--strict] [--local-assets] [--only <story> ...] [--story <id>] [--out <dir>] | ${name} check <story> [page id ...] [--places] | ${name} dev [--story <id>] [--out <dir>] [--port <n>] | ${name} i18n <lang> [--story id] | ${name} image <file or url> <name> --caption <text> [--credit <text>] [--source <url>] [--story <id>] | ${name} rehash | ${name} upload [--story <id>] [--dry-run] | ${name} deploy [--dry-run]${extra}`;
|
|
6
|
+
export const USAGE = (name = 'kitap', extra = '') => `usage: ${name} build [--strict] [--local-assets] [--only <story> ...] [--story <id>] [--out <dir>] | ${name} check <story> [page id ...] [--places] | ${name} dev [--story <id>] [--out <dir>] [--port <n>] | ${name} i18n <lang> [--story id] | ${name} image <file or url> <name> --caption <text> [--credit <text>] [--source <url>] [--story <id>] | ${name} image-candidates <story> --queries <file> [--limit <n>] [--pages <id,id>] [--out <name>] | ${name} rehash | ${name} upload [--story <id>] [--dry-run] | ${name} deploy [--dry-run]${extra}`;
|
|
7
7
|
|
|
8
8
|
// argv are the arguments after the command name. engine is the engine module to build with, else site.yaml names it.
|
|
9
9
|
// own(cmd, positionals, options) runs an engine's own command and returns true when it took it. options adds flags.
|
|
10
10
|
export async function run(argv, { name = 'kitap', engine = null, enginePath = null, own = null, options = {}, usage = '' } = {}) {
|
|
11
11
|
let args;
|
|
12
12
|
try {
|
|
13
|
-
args = parseArgs({ args: argv, allowPositionals: true, options: { strict: { type: 'boolean' }, 'local-assets': { type: 'boolean' }, 'dry-run': { type: 'boolean' }, places: { type: 'boolean' }, story: { type: 'string' }, caption: { type: 'string' }, credit: { type: 'string' }, source: { type: 'string' }, only: { type: 'string', multiple: true }, out: { type: 'string' }, port: { type: 'string' }, engine: { type: 'string' }, ...options } });
|
|
13
|
+
args = parseArgs({ args: argv, allowPositionals: true, options: { strict: { type: 'boolean' }, 'local-assets': { type: 'boolean' }, 'dry-run': { type: 'boolean' }, places: { type: 'boolean' }, story: { type: 'string' }, caption: { type: 'string' }, credit: { type: 'string' }, source: { type: 'string' }, only: { type: 'string', multiple: true }, out: { type: 'string' }, port: { type: 'string' }, queries: { type: 'string' }, limit: { type: 'string' }, pages: { type: 'string' }, engine: { type: 'string' }, ...options } });
|
|
14
14
|
} catch (err) { // an unknown or incomplete flag
|
|
15
15
|
console.error(`${err.message}\n${USAGE(name, usage)}`);
|
|
16
16
|
process.exit(2);
|
|
@@ -29,6 +29,7 @@ export async function run(argv, { name = 'kitap', engine = null, enginePath = nu
|
|
|
29
29
|
}
|
|
30
30
|
else if (cmd === 'i18n' && lang) i18n({ lang, story: opt.story, engine: e });
|
|
31
31
|
else if (cmd === 'image' && lang) await image({ file: lang, name: rest[0], caption: opt.caption, credit: opt.credit, source: opt.source, story: opt.story });
|
|
32
|
+
else if (cmd === 'image-candidates' && lang) await imageCandidates({ story: lang, queries: opt.queries, limit: Number(opt.limit) || 6, pages: opt.pages?.split(',') ?? null, out: opt.out ?? '' });
|
|
32
33
|
else if (cmd === 'rehash') rehash();
|
|
33
34
|
else if (cmd === 'upload') await upload({ story: opt.story, dryRun: opt['dry-run'], engine: e });
|
|
34
35
|
else if (cmd === 'deploy') await deploy({ dryRun: opt['dry-run'] });
|
package/src/index.mjs
CHANGED
|
@@ -4,6 +4,7 @@ export { build, buildPages, checkStory as check, readStory, storyAssets, uiCatal
|
|
|
4
4
|
export { dev } from './dev.mjs';
|
|
5
5
|
export { i18n } from './translate.mjs';
|
|
6
6
|
export { image, imageIndex, findImage, rehash } from './image.mjs';
|
|
7
|
+
export { imageCandidates } from './candidates.mjs';
|
|
7
8
|
export { upload } from './upload.mjs';
|
|
8
9
|
export { deploy } from './deploy.mjs';
|
|
9
10
|
export { resolve } from './routes.mjs';
|