@openbooknetwork/kitap 0.1.0 → 0.3.1

package/README.md

@@ -26,6 +26,7 @@ title: Open Body Book
 engine: beden
 languages: [en, tr]
 assets: { url: https://assets.example.org/, bucket: body-assets, endpoint: https://... }
+ask: [chatgpt, claude]               # optional: an Ask button on every page opens the assistant with the page's text
 ```
 
 ```yaml
@@ -49,6 +50,7 @@ kitap check <story> [page id ...] [--places]
 kitap dev [--story <id>] [--out <dir>] [--port <n>]
 kitap i18n <lang> [--story <id>]
 kitap image <file or url> <name> --caption <text> [--credit <text>] [--source <url>] [--story <id>]
+kitap image-candidates <story> --queries <file> [--limit <n>] [--pages <id,id>] [--out <name>]
 kitap rehash | kitap upload [--story <id>] [--dry-run] | kitap deploy [--dry-run]
 ```
 
@@ -56,6 +58,11 @@ An engine's own binary runs the same commands with its engine, `harita build` fo
 `--strict` fails on a missing translation or an object the store lacks; `--local-assets` shows the files on this
 machine before they are uploaded.
 
+`image-candidates` searches Wikimedia Commons for each page of a story and downloads what it finds to
+`<story>/.candidates/<page>/`, listed with author, license and source in `.candidates/index.yaml`. The queries file
+is keyed by story, then page: `{ queries: [search terms], articles: [Wikipedia titles or URLs], files: [Commons
+files] }`. A file the book holds already is listed with `held: <id>`. `kitap image <url>` adds a chosen one.
+
 ## An engine
 
 An engine is a package whose default export is a module with an id and hooks. Kitap resolves `engine: <id>` of
@@ -96,6 +103,24 @@ and, as a module, dispatches `kitap-engine` once set. `host` carries the bundle
 
 The test engine in `test/engines/plain.mjs` shows the least an engine needs.
 
+## Deploy
+
+A book serves from Cloudflare: the images and the engine's objects from an R2 bucket behind `assets.<zone>`, the
+pages from a second bucket through a Worker on the zone. `site.yaml` names them in `url`, `assets` and `pages`, and
+one script sets them up from a content project:
+
+```
+CF_TOKEN=<token> node node_modules/@openbooknetwork/kitap/scripts/cloudflare-book.mjs [--lock]
+```
+
+It adds the zone and prints its nameservers for the registrar, creates both buckets, attaches the assets domain,
+sets CORS, the cache rule, the SVG header rule and tiered cache, and with `--lock` the bucket lock on `images/`.
+Run it again once the zone is active. Then three tokens by hand: an R2 token on the assets bucket for `kitap
+upload`, one on the site bucket as the `AWS_ACCESS_KEY_ID` and `AWS_SECRET_ACCESS_KEY` secrets of the deploy
+workflow, and a wrangler token with Workers Scripts Edit and Workers Routes Edit as `CLOUDFLARE_API_TOKEN` with
+the account id as `CLOUDFLARE_ACCOUNT_ID`. `scripts/cloudflare-redirect.mjs <old zone> <new zone>` sends a
+retired domain to the new one.
+
 ## Work on it
 
 ```

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@openbooknetwork/kitap",
-  "version": "0.1.0",
+  "version": "0.3.1",
   "description": "The book under the Open Book Network sites: stories of pages with a text beside a picture an engine draws, built to a static site in every language",
   "keywords": [
     "static-site",
@@ -31,7 +31,8 @@
   },
   "files": [
     "bin",
-    "src"
+    "src",
+    "scripts"
   ],
   "publishConfig": {
     "access": "public"

package/scripts/cloudflare-book.mjs

@@ -0,0 +1,115 @@
+#!/usr/bin/env node
+// Sets a book up on Cloudflare from its site.yaml: the zone of `url`, the assets bucket with its custom domain and
+// CORS, the site bucket, the cache rule and the SVG header rule on the assets host, and tiered cache. Every step
+// is skipped when it is already done, so the script can run again after the zone goes active.
+//
+//   CF_TOKEN=<token> node node_modules/@openbooknetwork/kitap/scripts/cloudflare-book.mjs [--lock] [--dry-run]
+//
+// The token needs, on the account: Zone Edit, Workers R2 Storage Edit, DNS Edit; on the zone: Zone Read, Zone
+// Settings Edit, Cache Rules Edit, Transform Rules Edit. --lock adds the bucket lock on images/, for after the
+// first upload is checked. What stays manual: the nameservers at the registrar, and the R2 and wrangler tokens.
+import fs from 'node:fs';
+import { parseArgs } from 'node:util';
+import yaml from 'js-yaml';
+
+const API = 'https://api.cloudflare.com/client/v4';
+const CSP = "default-src 'none'; style-src 'unsafe-inline'; img-src data:";
+
+const { values } = parseArgs({ options: { lock: { type: 'boolean' }, 'dry-run': { type: 'boolean' }, account: { type: 'string' } } });
+const site = yaml.load(fs.readFileSync('site.yaml', 'utf8'));
+for (const k of ['url', 'assets', 'pages']) if (!site[k]) fail(`site.yaml has no ${k}, see kitap's README under Deploy`);
+const zoneName = new URL(site.url).hostname.replace(/^www\./, '');
+const assetsHost = new URL(site.assets.url).hostname;
+const assetsBucket = site.assets.bucket, siteBucket = site.pages.bucket;
+const token = process.env.CF_TOKEN, dry = values['dry-run'];
+if (!token && !dry) fail('set CF_TOKEN to a Cloudflare API token, see the head of this script for its permissions');
+
+console.log(`${zoneName}: assets ${assetsBucket} at ${assetsHost}, site ${siteBucket}`);
+if (dry) { console.log('dry run: nothing is sent'); process.exit(0); }
+
+const account = values.account ?? await accountId();
+const zone = await ensureZone();
+await ensureBucket(assetsBucket);
+await ensureBucket(siteBucket);
+await cors(assetsBucket);
+await managedDomainOff(assetsBucket);
+await customDomain(assetsBucket);
+await rules();
+if (values.lock) await lock(assetsBucket);
+console.log(zone.status === 'active' ? `${zoneName} is set up` : `${zoneName} is ${zone.status}: point the registrar at ${zone.name_servers.join(' and ')}, then run this again`);
+
+async function accountId() {
+  const list = await call('GET', '/accounts');
+  if (list.length !== 1) fail(`the token sees ${list.length} accounts, pass --account <id>`);
+  return list[0].id;
+}
+
+async function ensureZone() {
+  const found = await call('GET', `/zones?name=${zoneName}`);
+  if (found.length) { console.log(`zone ${zoneName}: ${found[0].status}`); return found[0]; }
+  const z = await call('POST', '/zones', { name: zoneName, account: { id: account }, type: 'full' });
+  console.log(`zone ${zoneName}: added, nameservers ${z.name_servers.join(' and ')}`);
+  return z;
+}
+
+async function ensureBucket(name) {
+  const res = await fetch(`${API}/accounts/${account}/r2/buckets/${name}`, { headers: { Authorization: `Bearer ${token}` } });
+  if (res.ok) { console.log(`bucket ${name}: exists`); return; }
+  await call('POST', `/accounts/${account}/r2/buckets`, { name });
+  console.log(`bucket ${name}: created`);
+}
+
+async function cors(bucket) {
+  await call('PUT', `/accounts/${account}/r2/buckets/${bucket}/cors`, { rules: [{ allowed: { origins: ['*'], methods: ['GET', 'HEAD'] } }] });
+  console.log(`bucket ${bucket}: CORS GET and HEAD from anywhere`);
+}
+
+// the r2.dev URL skips the cache, so the custom domain is the only way in
+async function managedDomainOff(bucket) {
+  await call('PUT', `/accounts/${account}/r2/buckets/${bucket}/domains/managed`, { enabled: false });
+}
+
+async function customDomain(bucket) {
+  const { domains } = await call('GET', `/accounts/${account}/r2/buckets/${bucket}/domains/custom`);
+  if (domains.some(d => d.domain === assetsHost)) { console.log(`bucket ${bucket}: ${assetsHost} attached`); return; }
+  if (zone.status !== 'active') { console.log(`bucket ${bucket}: ${assetsHost} waits for the zone to go active`); return; }
+  await call('POST', `/accounts/${account}/r2/buckets/${bucket}/domains/custom`, { domain: assetsHost, zoneId: zone.id, enabled: true, minTLS: '1.2' });
+  console.log(`bucket ${bucket}: ${assetsHost} attached`);
+}
+
+async function rules() {
+  await call('PUT', `/zones/${zone.id}/rulesets/phases/http_request_cache_settings/entrypoint`, { rules: [{
+    description: 'assets',
+    expression: `(http.host eq "${assetsHost}")`,
+    action: 'set_cache_settings',
+    action_parameters: {
+      cache: true,
+      cache_key: { custom_key: { query_string: { exclude: { all: true } } } },
+      // a 404 stays uncached, so a strict build right after an upload sees the new keys
+      edge_ttl: { mode: 'respect_origin', status_code_ttl: [{ status_code: 404, value: -1 }] },
+    },
+  }] });
+  await call('PUT', `/zones/${zone.id}/rulesets/phases/http_response_headers_transform/entrypoint`, { rules: [{
+    description: 'svg csp',
+    expression: `(http.host eq "${assetsHost}" and ends_with(http.request.uri.path, ".svg"))`,
+    action: 'rewrite',
+    action_parameters: { headers: { 'Content-Security-Policy': { operation: 'set', value: CSP } } },
+  }] });
+  await call('PATCH', `/zones/${zone.id}/cache/tiered_cache_smart_topology_enable`, { value: 'on' });
+  console.log(`zone ${zoneName}: cache rule, SVG header rule and tiered cache`);
+}
+
+// the store then refuses to delete or overwrite an image, whoever asks
+async function lock(bucket) {
+  await call('PUT', `/accounts/${account}/r2/buckets/${bucket}/lock`, { rules: [{ id: 'images', enabled: true, prefix: 'images/', condition: { type: 'Indefinite' } }] });
+  console.log(`bucket ${bucket}: images/ locked`);
+}
+
+async function call(method, path, body) {
+  const res = await fetch(API + path, { method, body: body && JSON.stringify(body), headers: { Authorization: `Bearer ${token}`, 'Content-Type': 'application/json' } });
+  const out = await res.json();
+  if (!out.success) fail(`${method} ${path} failed: ${JSON.stringify(out.errors, null, 2)}`);
+  return out.result;
+}
+
+function fail(message) { console.error(message); process.exit(1); }

package/scripts/cloudflare-redirect.mjs

@@ -0,0 +1,62 @@
+#!/usr/bin/env node
+// Sends an old zone to a new one by 301, the apex and the assets host alike, path and query kept:
+//
+//   CF_TOKEN=<token> node node_modules/@openbooknetwork/kitap/scripts/cloudflare-redirect.mjs openhistoryatlas.org openhistorybook.org [--dry-run]
+//
+// The token needs Zone Read and Single Redirect Edit on the old zone. The rule replaces the zone's own redirect
+// rules, so run it on a zone that has none or that this script made.
+import { parseArgs } from 'node:util';
+
+const API = 'https://api.cloudflare.com/client/v4';
+const { values, positionals } = parseArgs({ allowPositionals: true, options: { 'dry-run': { type: 'boolean' } } });
+const [from, to] = positionals;
+if (!from || !to) fail('usage: CF_TOKEN=<token> node cloudflare-redirect.mjs <old zone> <new zone> [--dry-run]');
+const token = process.env.CF_TOKEN;
+if (!token && !values['dry-run']) fail('set CF_TOKEN to a Cloudflare API token with Zone Read and Single Redirect Edit');
+
+const body = redirectRules(from, to);
+if (values['dry-run']) console.log(JSON.stringify(body, null, 2));
+else {
+  await call('PUT', `/zones/${await zoneId(from)}/rulesets/phases/http_request_dynamic_redirect/entrypoint`, body);
+  console.log(`${from} redirects to ${to}`);
+}
+
+// One rule per hostname: the apex to the new apex, the assets host to the new assets host, path and query kept.
+function redirectRules(from, to) {
+  const rule = (fromHost, toHost) => ({
+    description: `${fromHost} to ${toHost}`,
+    expression: `(http.host eq "${fromHost}")`,
+    action: 'redirect',
+    action_parameters: { from_value: {
+      status_code: 301,
+      target_url: { expression: `concat("https://${toHost}", http.request.uri.path)` },
+      preserve_query_string: true,
+    } },
+  });
+  return { rules: [rule(from, to), rule(`assets.${from}`, `assets.${to}`)] };
+}
+
+async function zoneId(name) {
+  const res = await fetch(`${API}/zones?name=${name}`, { headers: { Authorization: `Bearer ${token}` } });
+  const out = await res.json();
+  if (!out.success) fail(`zone lookup failed: ${JSON.stringify(out.errors, null, 2)}`);
+  if (!out.result.length) {
+    fail(`the token sees no zone named ${name}: give it Zone Read on that zone`);
+  }
+  return out.result[0].id;
+}
+
+async function call(method, path, body) {
+  const res = await fetch(API + path, {
+    method, body: JSON.stringify(body),
+    headers: { Authorization: `Bearer ${token}`, 'Content-Type': 'application/json' },
+  });
+  const out = await res.json();
+  if (!out.success) fail(`${method} ${path} failed: ${JSON.stringify(out.errors, null, 2)}`);
+  return out.result;
+}
+
+function fail(message) {
+  console.error(message);
+  process.exit(1);
+}

package/src/app.html

@@ -111,7 +111,11 @@ nav.pages button:focus-visible,nav.pages a.pg:focus-visible{outline:2px solid va
 .legend .r{width:1rem;height:0;border:0;border-top:2px solid var(--rc);position:relative}
 .legend .r.arrow::after{content:"";position:absolute;inset-inline-end:-2px;top:-5px;border:4px solid transparent;border-inline-start:5px solid var(--rc)}
 .legend .r.dashed{border-top-style:dashed}
-.credit{position:absolute;inset-inline-end:.714rem;bottom:.714rem;background:var(--panel);border:1px solid var(--rule);padding:3px .571rem;font-size:.786rem;color:var(--ink-2);pointer-events:auto;
+/* the corner of the map: the Ask button over the credit bar, and the popups they open above both */
+.corner{position:absolute;inset-inline-end:.714rem;bottom:.714rem;z-index:4;display:flex;flex-direction:column;align-items:flex-end;gap:.429rem;pointer-events:auto}
+.ask{display:flex;align-items:center;gap:.357rem;background:var(--panel);border:1px solid var(--rule);border-radius:6px;padding:.357rem .714rem;font:inherit;font-size:.929rem;font-weight:600;color:var(--ink);cursor:pointer}
+.ask svg{width:1.143rem;height:1.143rem} .ask[hidden]{display:none} .ask:focus-visible{outline:2px solid var(--accent)}
+.credit{background:var(--panel);border:1px solid var(--rule);padding:3px .571rem;font-size:.786rem;color:var(--ink-2);pointer-events:auto;
   display:flex;align-items:center;gap:.571rem}
 .credit a{color:inherit}
 .credit :is(#mapdata,#othersrc){background:none;border:0;border-inline-start:1px solid var(--rule);padding:0;padding-inline-start:.571rem;font:inherit;color:inherit;text-decoration:underline;cursor:pointer}
@@ -124,6 +128,7 @@ nav.pages button:focus-visible,nav.pages a.pg:focus-visible{outline:2px solid va
 .credits ul{margin:0;padding-inline-start:1rem;display:grid;gap:.286rem}
 .credit #othersrc[hidden]{display:none}
 .credits .made{display:none;margin:0 0 .429rem}
+.credits .ask-list{display:grid;gap:.286rem;margin:0 0 .429rem} .credits .note{margin:0;color:var(--ink-2)}
 /* map tools: the 2D | 3D switch at the top of the map, joined on phones by the Legend button */
 .maptools{position:absolute;top:.714rem;inset-inline-end:.714rem;z-index:3;display:flex;gap:.429rem;align-items:flex-start}
 .view{display:flex;background:var(--panel);border:1px solid var(--rule);border-radius:6px;overflow:hidden;box-shadow:0 1px 3px rgba(0,0,0,.15)}
@@ -253,7 +258,7 @@ aside.story figcaption small{display:block;font-size:.786rem;color:var(--ink-2);
   aside.story .text{grid-row:2;grid-column:1;z-index:2;background:var(--paper);overflow:auto;padding:.571rem .143rem 1.143rem}
   body[data-tab="map"] aside.story .text{display:none}
   /* in the text view the map's own overlays step aside; they sit above the text in the stacking order otherwise */
-  body[data-tab="text"] :is(.peek,.maptools,.legend,.credit,.maplibregl-popup,.hl-label){display:none}
+  body[data-tab="text"] :is(.peek,.maptools,.legend,.corner,.maplibregl-popup,.hl-label){display:none}
   .hl-label.pinned{top:calc(.571rem + max(40px,2.857rem) + .429rem)} /* below the map tools and the credit */
   /* the peek card: the first two lines of the page's text over the bottom of the map; a tap opens the text */
   .peek{display:block;position:absolute;inset-inline:.571rem;bottom:.571rem;z-index:3;background:var(--panel);border:1px solid var(--rule);border-radius:8px;
@@ -272,7 +277,9 @@ aside.story figcaption small{display:block;font-size:.786rem;color:var(--ink-2);
   .legend-btn[hidden]{display:none}
   .legend{top:calc(.571rem + max(40px,2.857rem) + .429rem);bottom:auto;inset-inline-start:.571rem;z-index:3;max-width:calc(100% - 1.143rem)}
   body:not(.legend-open) .legend{display:none}
-  .credit{top:.571rem;bottom:auto;inset-inline-end:.571rem;padding:0;border:0;background:none}
+  .corner{top:.571rem;bottom:auto;inset-inline-end:.571rem;flex-direction:row-reverse}
+  .ask{display:grid;place-items:center;width:max(44px,3.143rem);height:max(40px,2.857rem);padding:0;color:var(--ink-2)} .ask span{display:none}
+  .credit{padding:0;border:0;background:none}
   .credit #made{display:none}
   .credit :is(#mapdata,#othersrc){display:grid;place-items:center;width:max(44px,3.143rem);height:max(40px,2.857rem);padding:0;background:var(--panel);border:1px solid var(--rule);border-radius:6px;color:var(--ink-2)}
   .credit :is(#mapdata,#othersrc) svg{display:block} .credit :is(#mapdata,#othersrc) span{display:none}
@@ -338,7 +345,7 @@ aside.story figcaption small{display:block;font-size:.786rem;color:var(--ink-2);
     <div class="maptools"><button type="button" class="legend-btn" id="legend-btn"></button><div class="view" id="view" role="group"><button type="button" data-view="2d" aria-pressed="true"></button><button type="button" data-view="3d" aria-pressed="false"></button></div></div>
     <div class="legend" id="legend"></div>
     <div class="hl-label" id="hl-label" hidden></div>
-    <div class="credit" id="credit"><span id="made"></span><button type="button" id="mapdata" aria-expanded="false" aria-controls="credits"></button><div class="credits" id="credits" hidden></div><button type="button" id="othersrc" aria-expanded="false" aria-controls="othercredits" hidden></button><div class="credits" id="othercredits" hidden></div></div>
+    <div class="corner" id="corner"><button type="button" class="ask" id="ask" aria-expanded="false" aria-controls="askmenu" hidden></button><div class="credits" id="askmenu" hidden></div><div class="credit" id="credit"><span id="made"></span><button type="button" id="mapdata" aria-expanded="false" aria-controls="credits"></button><div class="credits" id="credits" hidden></div><button type="button" id="othersrc" aria-expanded="false" aria-controls="othercredits" hidden></button><div class="credits" id="othercredits" hidden></div></div></div>
     <div class="peek" id="peek" role="button" tabindex="0"><p id="peek-text"></p><span class="more" id="peek-more"></span><button type="button" class="close" id="peek-close">×</button></div>
     <div class="lightbox" id="lightbox" hidden>
       <figure><img id="lb-img" alt=""><figcaption id="lb-cap"></figcaption></figure>

package/src/app.js

@@ -63,6 +63,7 @@ function renderChrome(){
   $('mapdata').innerHTML = `${ICONS.info}<span>${esc(T().map_data)}</span>`; $('mapdata').setAttribute('aria-label', T().map_data);
   $('mapdata').hidden = !B.credits.length;
   $('othersrc').innerHTML = `${ICONS['external-link']}<span>${esc(T().other_sources)}</span>`; $('othersrc').setAttribute('aria-label', T().other_sources);
+  $('ask').innerHTML = `${ICONS['message-circle']}<span>${esc(T().ask)}</span>`; $('ask').setAttribute('aria-label', T().ask); $('ask').hidden = !B.site.ask.length;
   $('credits').innerHTML = `<p class="made">${credit(T())}</p><h4>${esc(T().map_data)}</h4><ul>${B.credits.map(c => `<li>${esc(L(c))}</li>`).join('')}</ul>`;
   $('prev').innerHTML = `${ICONS['chevron-left']} ${esc(T().prev)}`;
   $('next').innerHTML = `${esc(T().next)} ${ICONS['chevron-right']}`;
@@ -201,11 +202,24 @@ $('peek').onkeydown = e => { if (e.key === 'Enter' || e.key === ' ') { e.prevent
 $('peek-close').onclick = e => { e.stopPropagation(); $('peek').hidden = true; };
 $('legend-btn').onclick = () => document.body.classList.toggle('legend-open');
 // the data credit popup and the page's other sources popup share the credit bar, one open at a time
-const CREDIT_POPUPS = { mapdata: 'credits', othersrc: 'othercredits' };
+const CREDIT_POPUPS = { mapdata: 'credits', othersrc: 'othercredits', ask: 'askmenu' };
 const showCredits = (open, btn = 'mapdata') => { for (const [b, p] of Object.entries(CREDIT_POPUPS)) { const on = open && b === btn; $(p).hidden = !on; $(b).setAttribute('aria-expanded', on); } };
 $('mapdata').onclick = () => showCredits($('credits').hidden, 'mapdata');
 $('othersrc').onclick = () => showCredits($('othercredits').hidden, 'othersrc');
-document.addEventListener('click', e => { if (!e.target.closest('.credit')) showCredits(false); });
+// the assistants of site.yaml, each opened in a new tab with the page's text as the prompt; the reader's own account answers
+const ASSISTANTS = { chatgpt: { name: 'ChatGPT', url: q => `https://chatgpt.com/?q=${q}` }, claude: { name: 'Claude', url: q => `https://claude.ai/new?q=${q}` } };
+function askPrompt(p){
+  const d = document.createElement('div'); d.innerHTML = p === OVERVIEW ? `<p>${esc(L(B.summary))}</p>` : p.html[lang];
+  d.querySelectorAll('figure').forEach(f => f.remove()); // the prose only, without the image captions and credits
+  const text = d.textContent.replace(/\s+/g, ' ').trim().slice(0, 2500);
+  return `${T().ask_prompt.replace('{title}', L(p.title)).replace('{url}', location.href)}\n\n${text}`;
+}
+$('ask').onclick = () => {
+  const q = encodeURIComponent(askPrompt(here()));
+  $('askmenu').innerHTML = `<h4>${esc(T().ask_title)}</h4><div class="ask-list">${B.site.ask.map(id => `<a href="${esc(ASSISTANTS[id].url(q))}" target="_blank" rel="noopener">${esc(ASSISTANTS[id].name)} ${ICONS['external-link']}</a>`).join('')}</div><p class="note">${esc(T().ask_note)}</p>`;
+  showCredits($('askmenu').hidden, 'ask');
+};
+document.addEventListener('click', e => { if (!e.target.closest('.corner')) showCredits(false); });
 const showSettings = open => { $('settings').hidden = !open; $('settings-btn').setAttribute('aria-expanded', open); };
 $('settings-btn').onclick = () => showSettings($('settings').hidden);
 document.addEventListener('click', e => { if (!$('settings').hidden && !e.target.closest('.prefs')) showSettings(false); });

package/src/build.mjs

@@ -48,7 +48,7 @@ const readUi = dir => Object.fromEntries(fs.readdirSync(dir).filter(f => f.endsW
 const UI = readUi(UI_DIR);
 // Icons come from Lucide: the shell's own, and the ones an engine asks for by name
 const ICON_DIR = path.dirname(require.resolve('lucide-static/icons/flag.svg'));
-const SHELL_ICONS = ['x', 'external-link', 'chevron-left', 'chevron-right', 'list', 'info', 'settings', 'languages'];
+const SHELL_ICONS = ['x', 'external-link', 'chevron-left', 'chevron-right', 'list', 'info', 'settings', 'languages', 'message-circle'];
 function iconSvg(name, where) {
   const file = path.join(ICON_DIR, name + '.svg');
   if (!exists(file)) fail(where, `unknown icon "${name}", pick a name from https://lucide.dev/icons`);
@@ -375,7 +375,7 @@ function buildStory(p, storyDir, site, engine) {
   // --- bundle and pages ---
   const bundle = {
     id: story.id, title: T(story.title, 'story.title'), summary, span, cover, languages: langs, defaultLanguage: defaultLang, ui, families, engine: engine.id,
-    site: { title: siteT(site.title, 'site.title'), languages: site.langs, defaultLanguage: site.defaultLang,
+    site: { title: siteT(site.title, 'site.title'), languages: site.langs, defaultLanguage: site.defaultLang, ask: site.ask ?? [],
       source: site.repository ? `${site.repository.replace(/\/$/, '')}/tree/${site.branch}/${where.split(path.sep).join('/')}` : null },
     themes: themeList(site.themes, ui, langs), defaultTheme: story.theme ?? site.theme,
     credits: eb.credits ?? [], images, icons, pages: pages.map(({ dir, include, ...p }) => p), tree: navTree(tree),

package/src/candidates.mjs

@@ -0,0 +1,112 @@
+// kitap image-candidates: searches Wikimedia Commons for each page's queries and articles, and downloads what it finds
+// to <story folder>/.candidates/<page>/ for review, listed with author, license and source in .candidates/index.yaml.
+// kitap image then adds a chosen one by its url. A file the book holds already, by its Commons page or its bytes, is
+// listed with held: <id> and not kept.
+import fs from 'fs';
+import path from 'path';
+import yaml from 'js-yaml';
+import { storyFolders } from './folders.mjs';
+import { imageIndex, findImage } from './image.mjs';
+
+const API = 'https://commons.wikimedia.org/w/api.php';
+const BITMAP = /\.(jpe?g|png|tiff?)$/i;
+// an article is a Wikipedia URL or an English Wikipedia title
+const articleApi = a => { const m = a.match(/^https?:\/\/([a-z-]+)\.wikipedia\.org\/wiki\/([^?#]+)/); return m ? [`https://${m[1]}.wikipedia.org/w/api.php`, decodeURIComponent(m[2]).replace(/_/g, ' ')] : ['https://en.wikipedia.org/w/api.php', a]; };
+const strip = s => String(s ?? '').replace(/<[^>]+>/g, '').replace(/\s+/g, ' ').trim();
+
+// queries is a YAML file keyed by story id, then by page id: { queries: [search terms], articles: [titles or URLs],
+// files: [Commons file names] }, or a bare list of search terms. pages refetches those pages, out writes into
+// .candidates/<out>/ so agents fetching at the same time keep separate index files. pause scales the waits.
+export async function imageCandidates({ root = process.cwd(), story, queries, limit = 6, pages = null, out = '', log = console.log, fetch = globalThis.fetch, pause = 1 } = {}) {
+  const stories = storyFolders(path.join(root, 'content'), p => path.relative(root, p)); // full paths by folder name
+  if (!stories.has(story)) throw new Error(`"${story}" is not a story folder under content/`);
+  if (!queries) throw new Error('name the queries file with --queries <file>: YAML keyed by story, then page, each with queries, articles or files');
+  const spec = yaml.load(fs.readFileSync(path.resolve(root, queries), 'utf8'))?.[story];
+  if (!spec) throw new Error(`${queries}: no queries for the story "${story}"`);
+  const outDir = path.join(stories.get(story), '.candidates', out);
+  fs.mkdirSync(outDir, { recursive: true });
+
+  // Wikimedia asks for a contact in the user agent and throttles unnamed clients harder
+  const siteFile = path.join(root, 'site.yaml'), site = fs.existsSync(siteFile) ? yaml.load(fs.readFileSync(siteFile, 'utf8')) ?? {} : {};
+  const title = typeof site.title === 'object' ? Object.values(site.title)[0] : site.title;
+  const headers = { 'User-Agent': `${title ?? 'kitap'} image-candidates (${site.repository ?? site.url ?? 'https://github.com/openbooknetwork/kitap'})` };
+  const sleep = ms => new Promise(r => setTimeout(r, ms * pause));
+  // Commons answers 429 when asked too fast: wait as told, or doubling from two seconds, up to six tries
+  async function get(url) {
+    for (let attempt = 0; ; attempt++) {
+      // a dropped connection is retried like a 5xx
+      const res = await fetch(url, { headers }).catch(e => { if (attempt >= 6) throw e; return { ok: false, status: 599, headers: new Headers() }; });
+      if (res.ok) return res;
+      if ((res.status === 429 || res.status >= 500) && attempt < 6) {
+        const wait = Number(res.headers.get('retry-after')) * 1000 || 2000 * 2 ** attempt;
+        log(`  ${res.status}, waiting ${wait / 1000}s`); await sleep(wait); continue;
+      }
+      throw new Error(`${res.status} for ${url}`);
+    }
+  }
+  const api = async (params, base = API) => (await get(base + '?' + new URLSearchParams({ format: 'json', ...params }))).json();
+
+  // resume: pages already in index.yaml are kept and skipped
+  const indexFile = path.join(outDir, 'index.yaml');
+  const index = fs.existsSync(indexFile) ? yaml.load(fs.readFileSync(indexFile, 'utf8')) ?? {} : {};
+  const save = () => fs.writeFileSync(indexFile, yaml.dump(index, { lineWidth: 120 }));
+  const held = imageIndex({ root });
+  const seen = new Set(Object.values(index).flat().map(c => 'File:' + c.title));
+  for (const [page, s] of Object.entries(spec)) {
+    if (pages && !pages.includes(page)) continue;
+    if (pages && index[page]) { for (const c of index[page]) seen.delete('File:' + c.title); delete index[page]; fs.rmSync(path.join(outDir, page), { recursive: true, force: true }); }
+    if (index[page]) { log(`${page}: done earlier, skipping`); continue; }
+    const terms = Array.isArray(s) ? s : s.queries ?? [], files = Array.isArray(s) ? [] : s.files ?? [], articles = Array.isArray(s) ? [] : s.articles ?? [];
+    const titles = files.map(f => f.startsWith('File:') ? f : 'File:' + f);
+    // every bitmap an article uses; files local to Wikipedia, usually non-free, have no Commons record and drop out below
+    for (const a of articles) {
+      const [base, t] = articleApi(a);
+      const r = await api({ action: 'query', prop: 'images', titles: t, imlimit: 'max', redirects: 1 }, base);
+      const found = Object.values(r.query?.pages ?? {});
+      if (!found.length || found.some(p => 'missing' in p)) log(`  no article "${t}"`);
+      for (const p of found) for (const im of p.images ?? []) if (BITMAP.test(im.title)) titles.push(im.title);
+      await sleep(1000);
+    }
+    for (const q of terms) {
+      const r = await api({ action: 'query', list: 'search', srsearch: `${q} filetype:bitmap`, srnamespace: 6, srlimit: limit * 2 });
+      const hits = (r.query?.search ?? []).map(h => h.title).filter(t => BITMAP.test(t) && !seen.has(t) && !titles.includes(t));
+      titles.push(...hits.slice(0, limit));
+      await sleep(1000);
+    }
+    const picked = [...new Set(titles)].filter(t => !seen.has(t));
+    if (!picked.length) { log(`${page}: nothing found`); continue; }
+    const infoPages = [];
+    for (let i = 0; i < picked.length; i += 50) { // the API takes at most 50 titles per call
+      const info = await api({ action: 'query', prop: 'imageinfo', titles: picked.slice(i, i + 50).join('|'), iiprop: 'url|extmetadata|size', iiurlwidth: 1280 }); // a standard thumbnail step, Wikimedia throttles other widths
+      infoPages.push(...Object.values(info.query?.pages ?? {}));
+    }
+    const dir = path.join(outDir, page);
+    fs.mkdirSync(dir, { recursive: true });
+    index[page] = [];
+    let n = 0;
+    for (const p of infoPages) {
+      const ii = p.imageinfo?.[0]; if (!ii) continue;
+      const meta = ii.extmetadata ?? {};
+      // the extension comes from the downloaded thumbnail, so a long title cut to 80 characters keeps one
+      const url = ii.thumburl ?? ii.url, ext = path.extname(new URL(url).pathname).toLowerCase();
+      const base = p.title.replace(/^File:/, '').replace(/\.\w+$/, '').replace(/[^\w.-]+/g, '_').slice(0, 80);
+      const commonsPage = `https://commons.wikimedia.org/wiki/${encodeURIComponent(p.title)}`;
+      const entry = { title: p.title.replace(/^File:/, ''), page: commonsPage, url, author: strip(meta.Artist?.value), license: strip(meta.LicenseShortName?.value),
+        date: strip(meta.DateTimeOriginal?.value), description: strip(meta.ImageDescription?.value).slice(0, 300), width: ii.width, height: ii.height };
+      seen.add(p.title);
+      const known = findImage(held, { source: commonsPage });
+      if (known) { index[page].push({ file: null, held: known.id, ...entry }); log(`  held already: ${known.id}`); continue; }
+      const name = `${String(++n).padStart(2, '0')}-${base}${ext}`, file = path.join(dir, name);
+      try { fs.writeFileSync(file, Buffer.from(await (await get(url)).arrayBuffer())); } catch (e) { log(`  skip ${p.title}: ${e.message}`); continue; }
+      // the same bytes in an image folder that names no source: the download goes
+      const same = findImage(held, { file });
+      if (same) { fs.rmSync(file); index[page].push({ file: null, held: same.id, ...entry }); log(`  held already: ${same.id}`); continue; }
+      index[page].push({ file: name, ...entry });
+      await sleep(800);
+    }
+    log(`${page}: ${index[page].length} candidates`);
+    save();
+  }
+  log(`wrote ${path.relative(root, indexFile)}`);
+  return index;
+}

package/src/cli.mjs

@@ -1,16 +1,16 @@
 // The command line of kitap and of every engine: the book's commands with the options they take. An engine's binary
 // calls run with its engine and its own commands, which are tried first.
 import { parseArgs } from 'util';
-import { build, check, dev, i18n, image, rehash, upload, deploy } from './index.mjs';
+import { build, check, dev, i18n, image, imageCandidates, rehash, upload, deploy } from './index.mjs';
 
-export const USAGE = (name = 'kitap', extra = '') => `usage: ${name} build [--strict] [--local-assets] [--only <story> ...] [--story <id>] [--out <dir>] | ${name} check <story> [page id ...] [--places] | ${name} dev [--story <id>] [--out <dir>] [--port <n>] | ${name} i18n <lang> [--story id] | ${name} image <file or url> <name> --caption <text> [--credit <text>] [--source <url>] [--story <id>] | ${name} rehash | ${name} upload [--story <id>] [--dry-run] | ${name} deploy [--dry-run]${extra}`;
+export const USAGE = (name = 'kitap', extra = '') => `usage: ${name} build [--strict] [--local-assets] [--only <story> ...] [--story <id>] [--out <dir>] | ${name} check <story> [page id ...] [--places] | ${name} dev [--story <id>] [--out <dir>] [--port <n>] | ${name} i18n <lang> [--story id] | ${name} image <file or url> <name> --caption <text> [--credit <text>] [--source <url>] [--story <id>] | ${name} image-candidates <story> --queries <file> [--limit <n>] [--pages <id,id>] [--out <name>] | ${name} rehash | ${name} upload [--story <id>] [--dry-run] | ${name} deploy [--dry-run]${extra}`;
 
 // argv are the arguments after the command name. engine is the engine module to build with, else site.yaml names it.
 // own(cmd, positionals, options) runs an engine's own command and returns true when it took it. options adds flags.
 export async function run(argv, { name = 'kitap', engine = null, enginePath = null, own = null, options = {}, usage = '' } = {}) {
   let args;
   try {
-    args = parseArgs({ args: argv, allowPositionals: true, options: { strict: { type: 'boolean' }, 'local-assets': { type: 'boolean' }, 'dry-run': { type: 'boolean' }, places: { type: 'boolean' }, story: { type: 'string' }, caption: { type: 'string' }, credit: { type: 'string' }, source: { type: 'string' }, only: { type: 'string', multiple: true }, out: { type: 'string' }, port: { type: 'string' }, engine: { type: 'string' }, ...options } });
+    args = parseArgs({ args: argv, allowPositionals: true, options: { strict: { type: 'boolean' }, 'local-assets': { type: 'boolean' }, 'dry-run': { type: 'boolean' }, places: { type: 'boolean' }, story: { type: 'string' }, caption: { type: 'string' }, credit: { type: 'string' }, source: { type: 'string' }, only: { type: 'string', multiple: true }, out: { type: 'string' }, port: { type: 'string' }, queries: { type: 'string' }, limit: { type: 'string' }, pages: { type: 'string' }, engine: { type: 'string' }, ...options } });
   } catch (err) { // an unknown or incomplete flag
     console.error(`${err.message}\n${USAGE(name, usage)}`);
     process.exit(2);
@@ -29,6 +29,7 @@ export async function run(argv, { name = 'kitap', engine = null, enginePath = nu
     }
     else if (cmd === 'i18n' && lang) i18n({ lang, story: opt.story, engine: e });
     else if (cmd === 'image' && lang) await image({ file: lang, name: rest[0], caption: opt.caption, credit: opt.credit, source: opt.source, story: opt.story });
+    else if (cmd === 'image-candidates' && lang) await imageCandidates({ story: lang, queries: opt.queries, limit: Number(opt.limit) || 6, pages: opt.pages?.split(',') ?? null, out: opt.out ?? '' });
     else if (cmd === 'rehash') rehash();
     else if (cmd === 'upload') await upload({ story: opt.story, dryRun: opt['dry-run'], engine: e });
     else if (cmd === 'deploy') await deploy({ dryRun: opt['dry-run'] });

package/src/dev.mjs

@@ -12,7 +12,7 @@ const RELOAD = `<script>(()=>{let v=null;setInterval(async()=>{try{const r=await
 // Serve root/dist on the port and rebuild on changes: a change in one story's folder rebuilds that story and the main
 // page, any other change every story. Returns the http server. Builds run in a child process, so package edits apply,
 // and with --local-assets, so the images on this machine show before they are uploaded.
-// story serves that one story as a site of its own, built into out, by default dist/.
+// story serves that one story as a site of its own. Every build writes into out, by default dist/, which the server serves.
 export function dev({ root = process.cwd(), port = Number(process.env.PORT) || 8080, log = console.log, story = null, out = null, engine = null } = {}) {
   const DIST = out ? path.resolve(root, out) : path.join(root, 'dist'), CONTENT = path.join(root, 'content');
   // engine is the engine module's path, handed to each build and watched beside kitap's own source
@@ -25,7 +25,7 @@ export function dev({ root = process.cwd(), port = Number(process.env.PORT) || 8
     building = true;
     const t0 = Date.now(), wasAll = all, stories = [...changed];
     all = false; changed = new Set();
-    const scope = story ? ['--story', story, ...(out ? ['--out', out] : [])] : wasAll ? [] : stories.flatMap(s => ['--only', s]);
+    const scope = [...(story ? ['--story', story] : wasAll ? [] : stories.flatMap(s => ['--only', s])), ...(out ? ['--out', out] : [])];
     const p = spawn(process.execPath, [path.join(PKG, 'bin', 'kitap.mjs'), 'build', '--local-assets', ...(engine ? ['--engine', engine] : []), ...scope], { stdio: 'inherit', cwd: root });
     p.on('exit', code => {
       building = false;

package/src/i18n/en.yaml

@@ -28,6 +28,10 @@ search_stories: Search stories
 no_match: "No story matches “{q}”."
 map_data: Data
 other_sources: Other sources
+ask: Ask AI
+ask_title: Ask about this page
+ask_prompt: "I am reading “{title}” at {url}. Answer my questions about it from this text and the page's sources, and say when you are unsure."
+ask_note: Opens in a new tab with the page's text as the question.
 start: Start
 not_found: There is no page at this address.
 settings: Settings

package/src/i18n/tr.yaml

@@ -28,6 +28,10 @@ search_stories: Hikâye ara
 no_match: "“{q}” ile eşleşen hikâye yok."
 map_data: Veri
 other_sources: Diğer kaynaklar
+ask: Yapay zekâya sor
+ask_title: Bu sayfayı sor
+ask_prompt: "“{title}” sayfasını okuyorum: {url}. Sorularımı bu metne ve sayfanın kaynaklarına dayanarak yanıtla, emin olmadığında söyle."
+ask_note: Yeni sekmede, sayfanın metni soru olarak açılır.
 start: Başla
 not_found: Bu adreste bir sayfa yok.
 settings: Ayarlar

package/src/index.mjs

@@ -4,6 +4,7 @@ export { build, buildPages, checkStory as check, readStory, storyAssets, uiCatal
 export { dev } from './dev.mjs';
 export { i18n } from './translate.mjs';
 export { image, imageIndex, findImage, rehash } from './image.mjs';
+export { imageCandidates } from './candidates.mjs';
 export { upload } from './upload.mjs';
 export { deploy } from './deploy.mjs';
 export { resolve } from './routes.mjs';

package/src/schema.mjs

@@ -28,6 +28,7 @@ export const Site = z.object({
   themes: Themes.optional(),
   url: z.string().url().optional(), // where the site is published, for canonical links, sitemaps and llms.txt
   repository: z.string().url().optional(), // the content repository: a Contribute link on the main page, a source link per story
+  ask: z.array(z.enum(['chatgpt', 'claude'])).default([]), // assistants a page offers to open with its text as the prompt
   branch: z.string().default('main'),
   // the object store that holds the images and the engine's objects: url serves them, bucket and endpoint take uploads
   assets: z.object({ url: z.string().url().refine(u => u.endsWith('/'), 'ends in /'), bucket: z.string().min(1), endpoint: z.string().url() }).strict().optional(),