@mthanh/vespercli 2.2.0 → 2.3.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +6 -3
- package/assets/CATALOG.md +9 -2
- package/assets/claude/.claude-plugin/marketplace.json +9 -9
- package/assets/claude/.claude-plugin/plugin.json +1 -1
- package/assets/claude/agents/code-planner.md +1 -1
- package/assets/claude/agents/code-reviewer.md +1 -1
- package/assets/claude/agents/code-tester.md +1 -1
- package/assets/claude/agents/copywriter.md +1 -1
- package/assets/claude/agents/feature-wrap-document.md +1 -1
- package/assets/claude/agents/janitor.md +1 -1
- package/assets/claude/agents/modernization.md +1 -1
- package/assets/claude/agents/principal-software-engineer.md +1 -1
- package/assets/claude/agents/refine-issue.md +1 -1
- package/assets/claude/agents/se-system-architecture-reviewer.md +1 -1
- package/assets/claude/agents/se-technical-writer.md +1 -1
- package/assets/claude/agents/skill-creator.md +1 -1
- package/assets/claude/agents/task-researcher.md +1 -1
- package/assets/claude/agents/technical-product-manager.md +1 -1
- package/assets/claude/agents/vision-analyst.md +1 -1
- package/assets/claude/agents/vitest.md +1 -1
- package/assets/claude/agents/workspace-learner.md +2 -2
- package/assets/claude/packs/backend/.claude-plugin/plugin.json +1 -1
- package/assets/claude/packs/backend/agents/firebase-developer.md +1 -1
- package/assets/claude/packs/backend/agents/nodejs-express-developer.md +1 -1
- package/assets/claude/packs/docs/.claude-plugin/plugin.json +1 -1
- package/assets/claude/packs/frontend/.claude-plugin/plugin.json +1 -1
- package/assets/claude/packs/frontend/agents/expert-nextjs-developer.md +1 -1
- package/assets/claude/packs/frontend/agents/expert-react-frontend-engineer.md +1 -1
- package/assets/claude/packs/frontend/agents/jest-react-component-analyzer.md +1 -1
- package/assets/claude/packs/frontend/agents/jest-react-scenario-scanner.md +1 -1
- package/assets/claude/packs/frontend/agents/jest-react-test-generator.md +1 -1
- package/assets/claude/packs/frontend/agents/jest-react-test-planner.md +1 -1
- package/assets/claude/packs/frontend/skills/design-taste-frontend/LICENSE +21 -0
- package/assets/claude/packs/frontend/skills/design-taste-frontend/SKILL.md +529 -0
- package/assets/claude/packs/frontend/skills/design-taste-frontend/UPSTREAM.json +7 -0
- package/assets/claude/packs/frontend/skills/design-taste-frontend/assets/app.css +216 -0
- package/assets/claude/packs/frontend/skills/design-taste-frontend/assets/base.css +105 -0
- package/assets/claude/packs/frontend/skills/design-taste-frontend/assets/base.js +37 -0
- package/assets/claude/packs/frontend/skills/design-taste-frontend/assets/charts.js +617 -0
- package/assets/claude/packs/frontend/skills/design-taste-frontend/assets/scenes/stage.css +28 -0
- package/assets/claude/packs/frontend/skills/design-taste-frontend/assets/scenes/stage.js +133 -0
- package/assets/claude/packs/frontend/skills/design-taste-frontend/evals/evals.json +76 -0
- package/assets/claude/packs/frontend/skills/design-taste-frontend/library/app.json +1327 -0
- package/assets/claude/packs/frontend/skills/design-taste-frontend/library/directions.json +1078 -0
- package/assets/claude/packs/frontend/skills/design-taste-frontend/library/heroes.json +1086 -0
- package/assets/claude/packs/frontend/skills/design-taste-frontend/library/motions.json +92 -0
- package/assets/claude/packs/frontend/skills/design-taste-frontend/library/palettes.json +1485 -0
- package/assets/claude/packs/frontend/skills/design-taste-frontend/library/photos.json +7240 -0
- package/assets/claude/packs/frontend/skills/design-taste-frontend/library/sections.json +4429 -0
- package/assets/claude/packs/frontend/skills/design-taste-frontend/library/sites.json +8350 -0
- package/assets/claude/packs/frontend/skills/design-taste-frontend/library/type.json +2024 -0
- package/assets/claude/packs/frontend/skills/design-taste-frontend/references/app-screens.md +1199 -0
- package/assets/claude/packs/frontend/skills/design-taste-frontend/references/colour.md +141 -0
- package/assets/claude/packs/frontend/skills/design-taste-frontend/references/copy.md +177 -0
- package/assets/claude/packs/frontend/skills/design-taste-frontend/references/critique.md +182 -0
- package/assets/claude/packs/frontend/skills/design-taste-frontend/references/imagery.md +791 -0
- package/assets/claude/packs/frontend/skills/design-taste-frontend/references/layout.md +292 -0
- package/assets/claude/packs/frontend/skills/design-taste-frontend/references/motion.md +980 -0
- package/assets/claude/packs/frontend/skills/design-taste-frontend/references/research.md +356 -0
- package/assets/claude/packs/frontend/skills/design-taste-frontend/references/scenes.md +174 -0
- package/assets/claude/packs/frontend/skills/design-taste-frontend/references/tells.md +115 -0
- package/assets/claude/packs/frontend/skills/design-taste-frontend/references/type.md +170 -0
- package/assets/claude/packs/frontend/skills/design-taste-frontend/scripts/assets.mjs +865 -0
- package/assets/claude/packs/frontend/skills/design-taste-frontend/scripts/draw.mjs +584 -0
- package/assets/claude/packs/frontend/skills/design-taste-frontend/scripts/inspect.mjs +303 -0
- package/assets/claude/packs/frontend/skills/design-taste-frontend/scripts/lib/app-draw.mjs +203 -0
- package/assets/claude/packs/frontend/skills/design-taste-frontend/scripts/lib/cdp.mjs +248 -0
- package/assets/claude/packs/frontend/skills/design-taste-frontend/scripts/lib/probe.mjs +909 -0
- package/assets/claude/packs/frontend/skills/design-taste-frontend/scripts/lib/rules.mjs +243 -0
- package/assets/claude/packs/frontend/skills/design-taste-frontend/scripts/lint.mjs +119 -0
- package/assets/claude/packs/frontend/skills/design-taste-frontend/scripts/scenes.mjs +124 -0
- package/assets/claude/packs/graphify/.claude-plugin/plugin.json +1 -1
- package/assets/claude/packs/integrations/.claude-plugin/plugin.json +1 -1
- package/assets/claude/packs/playwright/.claude-plugin/plugin.json +1 -1
- package/assets/claude/packs/playwright/agents/playwright-component-analyzer.md +1 -1
- package/assets/claude/packs/playwright/agents/playwright-scenario-scanner.md +1 -1
- package/assets/claude/packs/playwright/agents/playwright-test-generator.md +1 -1
- package/assets/claude/packs/playwright/agents/playwright-test-planner.md +1 -1
- package/assets/claude/packs/playwright/agents/playwright-tester.md +1 -1
- package/assets/claude/packs/polyglot/.claude-plugin/plugin.json +1 -1
- package/assets/claude/packs/styles/.claude-plugin/plugin.json +1 -1
- package/assets/copilot/agents/code-planner.agent.md +1 -1
- package/assets/copilot/agents/code-reviewer.agent.md +1 -1
- package/assets/copilot/agents/code-tester.agent.md +1 -1
- package/assets/copilot/agents/copywriter.agent.md +1 -1
- package/assets/copilot/agents/feature-wrap-document.agent.md +1 -1
- package/assets/copilot/agents/janitor.agent.md +1 -1
- package/assets/copilot/agents/modernization.agent.md +1 -1
- package/assets/copilot/agents/principal-software-engineer.agent.md +1 -1
- package/assets/copilot/agents/refine-issue.agent.md +1 -1
- package/assets/copilot/agents/se-system-architecture-reviewer.agent.md +1 -1
- package/assets/copilot/agents/se-technical-writer.agent.md +1 -1
- package/assets/copilot/agents/skill-creator.agent.md +1 -1
- package/assets/copilot/agents/task-researcher.agent.md +1 -1
- package/assets/copilot/agents/technical-product-manager.agent.md +1 -1
- package/assets/copilot/agents/vision-analyst.agent.md +1 -1
- package/assets/copilot/agents/vitest.agent.md +1 -1
- package/assets/copilot/agents/workspace-learner.agent.md +2 -2
- package/assets/copilot/packs/backend/agents/firebase-developer.agent.md +1 -1
- package/assets/copilot/packs/backend/agents/nodejs-express-developer.agent.md +1 -1
- package/assets/copilot/packs/frontend/agents/expert-nextjs-developer.agent.md +1 -1
- package/assets/copilot/packs/frontend/agents/expert-react-frontend-engineer.agent.md +1 -1
- package/assets/copilot/packs/frontend/agents/jest-react-component-analyzer.agent.md +1 -1
- package/assets/copilot/packs/frontend/agents/jest-react-scenario-scanner.agent.md +1 -1
- package/assets/copilot/packs/frontend/agents/jest-react-test-generator.agent.md +1 -1
- package/assets/copilot/packs/frontend/agents/jest-react-test-planner.agent.md +1 -1
- package/assets/copilot/packs/frontend/skills/design-taste-frontend/LICENSE +21 -0
- package/assets/copilot/packs/frontend/skills/design-taste-frontend/SKILL.md +529 -0
- package/assets/copilot/packs/frontend/skills/design-taste-frontend/UPSTREAM.json +7 -0
- package/assets/copilot/packs/frontend/skills/design-taste-frontend/assets/app.css +216 -0
- package/assets/copilot/packs/frontend/skills/design-taste-frontend/assets/base.css +105 -0
- package/assets/copilot/packs/frontend/skills/design-taste-frontend/assets/base.js +37 -0
- package/assets/copilot/packs/frontend/skills/design-taste-frontend/assets/charts.js +617 -0
- package/assets/copilot/packs/frontend/skills/design-taste-frontend/assets/scenes/stage.css +28 -0
- package/assets/copilot/packs/frontend/skills/design-taste-frontend/assets/scenes/stage.js +133 -0
- package/assets/copilot/packs/frontend/skills/design-taste-frontend/evals/evals.json +76 -0
- package/assets/copilot/packs/frontend/skills/design-taste-frontend/library/app.json +1327 -0
- package/assets/copilot/packs/frontend/skills/design-taste-frontend/library/directions.json +1078 -0
- package/assets/copilot/packs/frontend/skills/design-taste-frontend/library/heroes.json +1086 -0
- package/assets/copilot/packs/frontend/skills/design-taste-frontend/library/motions.json +92 -0
- package/assets/copilot/packs/frontend/skills/design-taste-frontend/library/palettes.json +1485 -0
- package/assets/copilot/packs/frontend/skills/design-taste-frontend/library/photos.json +7240 -0
- package/assets/copilot/packs/frontend/skills/design-taste-frontend/library/sections.json +4429 -0
- package/assets/copilot/packs/frontend/skills/design-taste-frontend/library/sites.json +8350 -0
- package/assets/copilot/packs/frontend/skills/design-taste-frontend/library/type.json +2024 -0
- package/assets/copilot/packs/frontend/skills/design-taste-frontend/references/app-screens.md +1199 -0
- package/assets/copilot/packs/frontend/skills/design-taste-frontend/references/colour.md +141 -0
- package/assets/copilot/packs/frontend/skills/design-taste-frontend/references/copy.md +177 -0
- package/assets/copilot/packs/frontend/skills/design-taste-frontend/references/critique.md +182 -0
- package/assets/copilot/packs/frontend/skills/design-taste-frontend/references/imagery.md +791 -0
- package/assets/copilot/packs/frontend/skills/design-taste-frontend/references/layout.md +292 -0
- package/assets/copilot/packs/frontend/skills/design-taste-frontend/references/motion.md +980 -0
- package/assets/copilot/packs/frontend/skills/design-taste-frontend/references/research.md +356 -0
- package/assets/copilot/packs/frontend/skills/design-taste-frontend/references/scenes.md +174 -0
- package/assets/copilot/packs/frontend/skills/design-taste-frontend/references/tells.md +115 -0
- package/assets/copilot/packs/frontend/skills/design-taste-frontend/references/type.md +170 -0
- package/assets/copilot/packs/frontend/skills/design-taste-frontend/scripts/assets.mjs +865 -0
- package/assets/copilot/packs/frontend/skills/design-taste-frontend/scripts/draw.mjs +584 -0
- package/assets/copilot/packs/frontend/skills/design-taste-frontend/scripts/inspect.mjs +303 -0
- package/assets/copilot/packs/frontend/skills/design-taste-frontend/scripts/lib/app-draw.mjs +203 -0
- package/assets/copilot/packs/frontend/skills/design-taste-frontend/scripts/lib/cdp.mjs +248 -0
- package/assets/copilot/packs/frontend/skills/design-taste-frontend/scripts/lib/probe.mjs +909 -0
- package/assets/copilot/packs/frontend/skills/design-taste-frontend/scripts/lib/rules.mjs +243 -0
- package/assets/copilot/packs/frontend/skills/design-taste-frontend/scripts/lint.mjs +119 -0
- package/assets/copilot/packs/frontend/skills/design-taste-frontend/scripts/scenes.mjs +124 -0
- package/assets/copilot/packs/playwright/agents/playwright-component-analyzer.agent.md +1 -1
- package/assets/copilot/packs/playwright/agents/playwright-scenario-scanner.agent.md +1 -1
- package/assets/copilot/packs/playwright/agents/playwright-test-generator.agent.md +1 -1
- package/assets/copilot/packs/playwright/agents/playwright-test-planner.agent.md +1 -1
- package/assets/copilot/packs/playwright/agents/playwright-tester.agent.md +1 -1
- package/assets/manifest.json +2 -0
- package/dist/cli.min.js +102 -102
- package/package.json +1 -1
- package/scripts/__snapshots__/generate-claude-assets.test.js.snap +1 -1
- package/scripts/generate-claude-assets.strict.test.js +1 -1
- package/scripts/generate-claude-assets.test.js +6 -6
- package/scripts/packs-layout.test.js +2 -1
|
@@ -0,0 +1,865 @@
|
|
|
1
|
+
#!/usr/bin/env node
|
|
2
|
+
// tasteskill / assets: find real, openly licensed pictures of the subject's world, and keep their credits.
|
|
3
|
+
//
|
|
4
|
+
// A page without photographs of its own does not have to draw everything. Archives, museums and open
|
|
5
|
+
// collections hold the subject's city and street, its maps, its materials, its trade's history and the
|
|
6
|
+
// museum objects of its kind, free to use with a credit. This searches them without keys and prints what
|
|
7
|
+
// an agent needs to use a picture honestly: a direct image URL, the page it comes from, its size, its
|
|
8
|
+
// creator, its licence and a one-line credit. Then it downloads the chosen ones into the project and
|
|
9
|
+
// records their credits, so the page works offline and credits every picture.
|
|
10
|
+
//
|
|
11
|
+
// node assets.mjs "<query>" search every source, licences safe for commercial use
|
|
12
|
+
// [--source all|openverse|commons|aic|met|museums] one or several, comma-separated
|
|
13
|
+
// [--license safe|pd|all|cc0,pd,by,by-sa,...] safe = CC0, public domain, CC BY, CC BY-SA
|
|
14
|
+
// [--orientation landscape|portrait|square] [--min-width 1000] [--width 1920] [--limit 5]
|
|
15
|
+
// [--no-check] skip checking that each image loads (faster)
|
|
16
|
+
// [--json] machine-readable results
|
|
17
|
+
// node assets.mjs "<query>" --video search for real footage instead (same options, plus
|
|
18
|
+
// [--res 720] the largest rendition to pick, by its short side
|
|
19
|
+
// [--max-mb 25] clips over this are flagged, and --get refuses them
|
|
20
|
+
// node assets.mjs --get <ref>[,<ref>...] [--to assets] [--name <file name>] [--width 1920] [--res 720] [--max-mb 25]
|
|
21
|
+
// download into the project and record the credit
|
|
22
|
+
// node assets.mjs --credits [--to assets] print the credits for the footer and the report
|
|
23
|
+
//
|
|
24
|
+
// Image sources: Openverse (api.openverse.org: Flickr, Wikimedia Commons, museums and libraries in one
|
|
25
|
+
// index), Wikimedia Commons, the Art Institute of Chicago (public-domain works over IIIF) and The
|
|
26
|
+
// Metropolitan Museum of Art (Open Access). Video sources: Wikimedia Commons (with its web-sized webm
|
|
27
|
+
// transcodes), the Internet Archive (its curated Prelinger Archives of films, public domain or Creative
|
|
28
|
+
// Commons) and the NASA Image and Video Library. Zero dependencies, no API keys. Node 18+ (built-in fetch).
|
|
29
|
+
import { readFileSync, writeFileSync, existsSync, mkdirSync, createWriteStream, unlinkSync } from 'node:fs';
|
|
30
|
+
import { join, resolve, relative } from 'node:path';
|
|
31
|
+
import { fileURLToPath } from 'node:url';
|
|
32
|
+
|
|
33
|
+
const UA = 'taste-skill-assets/2.0 (+https://github.com/Leonxlnx/taste-skill)';
|
|
34
|
+
const SELF = fileURLToPath(import.meta.url).split('\\').join('/');
|
|
35
|
+
const CMD = `node ${/\s/.test(SELF) ? `"${SELF}"` : SELF}`;
|
|
36
|
+
const SOURCES = {
|
|
37
|
+
openverse: { media: 'image', name: 'Openverse', note: 'one index over Flickr, Wikimedia Commons, museums and libraries' },
|
|
38
|
+
commons: { media: 'both', name: 'Wikimedia Commons', note: 'archival photographs, old maps, scans of prints and books, places as they look now', videoNote: 'video files with web-sized webm transcodes: places, crafts, processes, nature' },
|
|
39
|
+
aic: { media: 'image', name: 'Art Institute of Chicago', note: 'public-domain paintings, prints, photographs and textiles, served large' },
|
|
40
|
+
met: { media: 'image', name: 'The Metropolitan Museum of Art', note: 'public-domain objects, prints, drawings and photographs (Open Access)' },
|
|
41
|
+
archive: { media: 'video', name: 'Internet Archive', note: 'the Prelinger Archives: industrial, educational, city and home films from the 1920s to the 1970s, mostly long' },
|
|
42
|
+
nasa: { media: 'video', name: 'NASA Image and Video Library', note: 'footage of the Earth, space, aircraft, launches and laboratories' },
|
|
43
|
+
};
|
|
44
|
+
const SOURCE_ALIAS = { museums: ['aic', 'met'], wikimedia: ['commons'], artic: ['aic'], chicago: ['aic'], metmuseum: ['met'], ia: ['archive'], internetarchive: ['archive'] };
|
|
45
|
+
const VIDEO_TYPES = { 'video/mp4': 'mp4', 'video/webm': 'webm' }; // what every current browser plays; Ogg Theora and QuickTime are left out
|
|
46
|
+
const LICENCES = ['cc0', 'pd', 'by', 'by-sa', 'by-nc', 'by-nd', 'by-nc-sa', 'by-nc-nd', 'other'];
|
|
47
|
+
const SAFE = ['cc0', 'pd', 'by', 'by-sa'];
|
|
48
|
+
const LICENCE_ALIAS = { pdm: 'pd', 'public-domain': 'pd', publicdomain: 'pd', 'cc-by': 'by', 'cc-by-sa': 'by-sa', 'cc-zero': 'cc0' };
|
|
49
|
+
const WEB_TYPES = { 'image/jpeg': 'jpg', 'image/png': 'png', 'image/webp': 'webp', 'image/gif': 'gif', 'image/avif': 'avif', 'image/svg+xml': 'svg' };
|
|
50
|
+
const PROVIDERS = { flickr: 'Flickr', wikimedia: 'Wikimedia Commons', met: 'The Metropolitan Museum of Art', nypl: 'New York Public Library', rawpixel: 'rawpixel', stocksnap: 'StockSnap', clevelandmuseum: 'Cleveland Museum of Art', brooklynmuseum: 'Brooklyn Museum', sciencemuseum: 'Science Museum Group', europeana: 'Europeana', geographorguk: 'Geograph Britain and Ireland', inaturalist: 'iNaturalist', wordpress: 'WordPress Photo Directory' };
|
|
51
|
+
const AIC_FIELDS = ['id', 'title', 'artist_title', 'artist_display', 'date_display', 'image_id', 'thumbnail', 'medium_display', 'artwork_type_title', 'is_public_domain'];
|
|
52
|
+
const AIC_HEADERS = { 'AIC-User-Agent': UA };
|
|
53
|
+
|
|
54
|
+
const USAGE = `Find real, openly licensed pictures of the subject's world, download them, keep their credits.
|
|
55
|
+
|
|
56
|
+
${CMD} "<query>" [options]
|
|
57
|
+
--source all|openverse|commons|aic|met|museums comma-separated; default all
|
|
58
|
+
--license safe|pd|all|cc0,pd,by,by-sa,... default safe: CC0, public domain, CC BY, CC BY-SA
|
|
59
|
+
--orientation landscape|portrait|square
|
|
60
|
+
--min-width <px> smallest acceptable full width, default 1000
|
|
61
|
+
--width <px> width of the image URL to use or download, default 1920
|
|
62
|
+
--limit <n> results per source, default 5 (at most 20)
|
|
63
|
+
--no-check do not check that every image loads (faster, less reliable)
|
|
64
|
+
--json machine-readable results
|
|
65
|
+
${CMD} "<query>" --video [options] real footage: Wikimedia Commons, Internet Archive, NASA
|
|
66
|
+
--source all|commons|archive|nasa
|
|
67
|
+
--res <p> largest rendition to pick, by its short side, default 720
|
|
68
|
+
--max-mb <n> clips over this are flagged and --get refuses them, default 25
|
|
69
|
+
--min-width <px> default 640 for video
|
|
70
|
+
${CMD} --get <ref>[,<ref>...] [--to assets] [--name <file name>] [--width 1920] [--res 720] [--max-mb 25]
|
|
71
|
+
${CMD} --credits [--to assets]
|
|
72
|
+
|
|
73
|
+
Queries work best in the subject's own words and a medium: "ghent postcard", "ghent old map", "azulejo tiles",
|
|
74
|
+
"botanical engraving fig", "rotterdam harbour 1920", "basalt texture". The museum sources match one or two
|
|
75
|
+
words best ("fig", "harbour"). For footage, name the action or the process: "coffee roasting", "timber
|
|
76
|
+
construction", "river", "city street night".`;
|
|
77
|
+
|
|
78
|
+
// ---------- arguments ----------
|
|
79
|
+
const VALUED = new Set(['source', 'sources', 'license', 'licence', 'orientation', 'min-width', 'width', 'limit', 'get', 'to', 'name', 'type', 'res', 'max-mb']);
|
|
80
|
+
const FLAGS = new Set(['no-check', 'json', 'credits', 'help', 'video']);
|
|
81
|
+
const args = {}; const words = [];
|
|
82
|
+
const argv = process.argv.slice(2);
|
|
83
|
+
for (let i = 0; i < argv.length; i++) {
|
|
84
|
+
const a = argv[i];
|
|
85
|
+
if (a === '-h') { args.help = true; continue; }
|
|
86
|
+
if (!a.startsWith('--')) { words.push(a); continue; }
|
|
87
|
+
const eq = a.indexOf('=');
|
|
88
|
+
const key = a.slice(2, eq === -1 ? undefined : eq);
|
|
89
|
+
if (VALUED.has(key)) {
|
|
90
|
+
const v = eq === -1 ? argv[++i] : a.slice(eq + 1);
|
|
91
|
+
if (v === undefined || v === '' || (eq === -1 && v.startsWith('--'))) fail(`--${key} needs a value.`);
|
|
92
|
+
args[key] = v;
|
|
93
|
+
} else if (FLAGS.has(key)) args[key] = true;
|
|
94
|
+
else fail(`Unknown option --${key}.`);
|
|
95
|
+
}
|
|
96
|
+
function fail(message) { console.error(`${message}\n\n${USAGE}`); process.exit(2); }
|
|
97
|
+
function number(key, def, min, max) {
|
|
98
|
+
if (args[key] === undefined) return def;
|
|
99
|
+
const n = Number(args[key]);
|
|
100
|
+
if (!Number.isFinite(n) || n < min || n > max) fail(`--${key} must be a number from ${min} to ${max}.`);
|
|
101
|
+
return Math.round(n);
|
|
102
|
+
}
|
|
103
|
+
if (args.type !== undefined && !['image', 'images', 'video', 'videos'].includes(String(args.type).toLowerCase())) fail('--type is image or video.');
|
|
104
|
+
const VIDEO = Boolean(args.video) || /^videos?$/i.test(String(args.type || ''));
|
|
105
|
+
function pickSources(value) {
|
|
106
|
+
const media = VIDEO ? 'video' : 'image';
|
|
107
|
+
const usable = Object.keys(SOURCES).filter((k) => SOURCES[k].media === media || SOURCES[k].media === 'both');
|
|
108
|
+
const out = [];
|
|
109
|
+
for (const s of String(value).toLowerCase().split(',').map((x) => x.trim()).filter(Boolean)) {
|
|
110
|
+
const found = s === 'all' ? usable : SOURCE_ALIAS[s] || (SOURCES[s] ? [s] : null);
|
|
111
|
+
if (!found) fail(`Unknown source "${s}". ${VIDEO ? 'Video' : 'Image'} sources: ${usable.join(', ')}${VIDEO ? '' : ', museums (aic and met)'}, all.`);
|
|
112
|
+
const wrong = found.filter((x) => !usable.includes(x));
|
|
113
|
+
if (wrong.length) fail(`${wrong.join(', ')} has no ${media === 'video' ? 'video' : 'still images'} here. ${VIDEO ? 'Video' : 'Image'} sources: ${usable.join(', ')}.`);
|
|
114
|
+
for (const x of found) if (!out.includes(x)) out.push(x);
|
|
115
|
+
}
|
|
116
|
+
return out.length ? out : usable;
|
|
117
|
+
}
|
|
118
|
+
function pickLicences(value) {
|
|
119
|
+
const v = String(value).toLowerCase().trim();
|
|
120
|
+
if (v === 'safe') return SAFE;
|
|
121
|
+
if (v === 'pd') return ['cc0', 'pd'];
|
|
122
|
+
if (v === 'all' || v === 'any') return LICENCES;
|
|
123
|
+
const out = v.split(',').map((x) => x.trim()).filter(Boolean).map((x) => LICENCE_ALIAS[x] || x);
|
|
124
|
+
const bad = out.filter((x) => !LICENCES.includes(x));
|
|
125
|
+
if (bad.length || !out.length) fail(`Unknown licence "${bad.join(', ') || v}". Use safe, pd, all, or a list of: ${LICENCES.join(', ')}.`);
|
|
126
|
+
return out;
|
|
127
|
+
}
|
|
128
|
+
function pickOrientation(value) {
|
|
129
|
+
if (value === undefined) return null;
|
|
130
|
+
const v = String(value).toLowerCase();
|
|
131
|
+
const o = { landscape: 'landscape', wide: 'landscape', horizontal: 'landscape', portrait: 'portrait', tall: 'portrait', vertical: 'portrait', square: 'square' }[v];
|
|
132
|
+
if (!o) fail('--orientation is landscape, portrait or square.');
|
|
133
|
+
return o;
|
|
134
|
+
}
|
|
135
|
+
const o = {
|
|
136
|
+
sources: pickSources(args.source ?? args.sources ?? 'all'),
|
|
137
|
+
licences: pickLicences(args.license ?? args.licence ?? 'safe'),
|
|
138
|
+
orientation: pickOrientation(args.orientation),
|
|
139
|
+
minWidth: number('min-width', VIDEO ? 640 : 1000, 0, 20000),
|
|
140
|
+
width: number('width', 1920, 200, 4000),
|
|
141
|
+
limit: number('limit', 5, 1, 20),
|
|
142
|
+
video: VIDEO,
|
|
143
|
+
res: number('res', 720, 144, 2160),
|
|
144
|
+
maxMb: number('max-mb', 25, 1, 2000),
|
|
145
|
+
check: !args['no-check'],
|
|
146
|
+
to: resolve(String(args.to || 'assets')),
|
|
147
|
+
toArg: args.to ? ` --to ${args.to}` : '',
|
|
148
|
+
};
|
|
149
|
+
|
|
150
|
+
// ---------- small helpers ----------
|
|
151
|
+
const sleep = (ms) => new Promise((r) => setTimeout(r, ms));
|
|
152
|
+
const short = (s, n) => { const t = String(s || ''); return t.length > n ? t.slice(0, n - 1).replace(/[\s,;:.-]+$/, '') + '…' : t; };
|
|
153
|
+
const noQuery = (u) => (u ? String(u).split('?')[0] : u);
|
|
154
|
+
const rel = (p) => relative(process.cwd(), p).split('\\').join('/') || '.';
|
|
155
|
+
const size = (bytes) => (bytes >= 1048576 ? `${(bytes / 1048576).toFixed(1)} MB` : `${Math.max(1, Math.round(bytes / 1024))} KB`);
|
|
156
|
+
const shape = (w, h) => (!w || !h ? null : w / h >= 1.15 ? 'landscape' : w / h <= 0.87 ? 'portrait' : 'square');
|
|
157
|
+
const esc = (s) => String(s ?? '').replace(/&/g, '&').replace(/</g, '<').replace(/>/g, '>').replace(/"/g, '"');
|
|
158
|
+
function text(v) {
|
|
159
|
+
if (v === undefined || v === null) return '';
|
|
160
|
+
return String(v).replace(/<(style|script)\b[\s\S]*?<\/\1>/gi, ' ').replace(/<[^>]*>/g, ' ').replace(/ /g, ' ').replace(/"/g, '"').replace(/�?39;|'/g, "'")
|
|
161
|
+
.replace(/</g, '<').replace(/>/g, '>').replace(/&#(\d+);/g, (_, n) => String.fromCodePoint(Number(n))).replace(/&/g, '&')
|
|
162
|
+
.replace(/\s+/g, ' ').trim();
|
|
163
|
+
}
|
|
164
|
+
function keyOf(u) { let s = noQuery(u) || ''; try { s = decodeURIComponent(s); } catch {} return s.toLowerCase().replace(/^https?:\/\//, '').replace(/ /g, '_'); }
|
|
165
|
+
function slug(s) {
|
|
166
|
+
return String(s || '').normalize('NFKD').replace(/[\u0300-\u036f]/g, '').toLowerCase().replace(/[^a-z0-9]+/g, '-').replace(/^-+|-+$/g, '')
|
|
167
|
+
.split('-').slice(0, 7).join('-').slice(0, 60).replace(/-+$/, '');
|
|
168
|
+
}
|
|
169
|
+
// The words of a query that carry meaning, and whether a record's own description holds them
|
|
170
|
+
// (all of them, or all but one of three or more). Used to keep the museum sources on the subject.
|
|
171
|
+
function queryWords(q) { return String(q).toLowerCase().split(/[^\p{L}\p{N}]+/u).filter((w) => w && !['the', 'a', 'an', 'of', 'in', 'on', 'at', 'and', 'or', 'for', 'with', 'from', 'to', 'by'].includes(w)); }
|
|
172
|
+
function onSubject(wanted, description) {
|
|
173
|
+
if (!wanted.length) return true;
|
|
174
|
+
const loose = (s) => String(s).toLowerCase().normalize('NFKD').replace(/\p{M}/gu, '').replace(/our\b/g, 'or'); // harbour = harbor
|
|
175
|
+
const hay = loose(description);
|
|
176
|
+
const hits = wanted.filter((w) => { const x = loose(w); return hay.includes(x) || (x.length > 4 && hay.includes(x.replace(/(es|s)$/, ''))); }).length;
|
|
177
|
+
return hits >= (wanted.length >= 3 ? wanted.length - 1 : wanted.length);
|
|
178
|
+
}
|
|
179
|
+
async function mapLimit(items, n, fn) {
|
|
180
|
+
const out = new Array(items.length); let next = 0;
|
|
181
|
+
await Promise.all(Array.from({ length: Math.min(n, items.length) }, async () => { while (next < items.length) { const k = next++; out[k] = await fn(items[k], k); } }));
|
|
182
|
+
return out;
|
|
183
|
+
}
|
|
184
|
+
async function getJson(url, ms = 20000, headers = {}) {
|
|
185
|
+
for (let attempt = 1; ; attempt++) {
|
|
186
|
+
let r;
|
|
187
|
+
try { r = await fetch(url, { headers: { 'User-Agent': UA, Accept: 'application/json', ...headers }, signal: AbortSignal.timeout(ms) }); }
|
|
188
|
+
catch (e) {
|
|
189
|
+
const timedOut = e.name === 'TimeoutError' || e.name === 'AbortError';
|
|
190
|
+
if (!timedOut && attempt < 2) { await sleep(800); continue; }
|
|
191
|
+
throw new Error(timedOut ? `no answer within ${Math.round(ms / 1000)} s` : `could not connect (${e.cause?.code || e.message})`);
|
|
192
|
+
}
|
|
193
|
+
if ([502, 503, 504].includes(r.status) && attempt < 2) { await sleep(1200); continue; }
|
|
194
|
+
if (r.status === 429) throw new Error('rate limited (HTTP 429): wait a minute, or search another --source');
|
|
195
|
+
if (r.status === 410) throw new Error('this API endpoint has been retired (HTTP 410); assets.mjs needs updating for it');
|
|
196
|
+
if (!r.ok) throw new Error(`HTTP ${r.status}${r.status >= 500 ? ', the service is having trouble' : ''}`);
|
|
197
|
+
try { return await r.json(); } catch { throw new Error('the answer was not JSON'); }
|
|
198
|
+
}
|
|
199
|
+
}
|
|
200
|
+
// Width and height from the first bytes of a JPEG, PNG, GIF or WebP file.
|
|
201
|
+
function dims(b) {
|
|
202
|
+
if (!b || b.length < 26) return {};
|
|
203
|
+
if (b[0] === 0x89 && b.toString('ascii', 1, 4) === 'PNG') return { w: b.readUInt32BE(16), h: b.readUInt32BE(20) };
|
|
204
|
+
if (b.toString('ascii', 0, 3) === 'GIF') return { w: b.readUInt16LE(6), h: b.readUInt16LE(8) };
|
|
205
|
+
if (b.toString('ascii', 0, 4) === 'RIFF' && b.toString('ascii', 8, 12) === 'WEBP' && b.length >= 30) {
|
|
206
|
+
const f = b.toString('ascii', 12, 16);
|
|
207
|
+
if (f === 'VP8X') return { w: 1 + b.readUIntLE(24, 3), h: 1 + b.readUIntLE(27, 3) };
|
|
208
|
+
if (f === 'VP8L') { const bits = b.readUInt32LE(21); return { w: 1 + (bits & 0x3fff), h: 1 + ((bits >>> 14) & 0x3fff) }; }
|
|
209
|
+
if (f === 'VP8 ') return { w: b.readUInt16LE(26) & 0x3fff, h: b.readUInt16LE(28) & 0x3fff };
|
|
210
|
+
}
|
|
211
|
+
if (b[0] === 0xff && b[1] === 0xd8) {
|
|
212
|
+
let i = 2;
|
|
213
|
+
while (i + 9 < b.length) {
|
|
214
|
+
if (b[i] !== 0xff) { i++; continue; }
|
|
215
|
+
const m = b[i + 1];
|
|
216
|
+
if (m === 0xff) { i++; continue; }
|
|
217
|
+
if (m === 0x01 || (m >= 0xd0 && m <= 0xd8)) { i += 2; continue; }
|
|
218
|
+
if (m >= 0xc0 && m <= 0xcf && m !== 0xc4 && m !== 0xc8 && m !== 0xcc) return { w: b.readUInt16BE(i + 7), h: b.readUInt16BE(i + 5) };
|
|
219
|
+
i += 2 + b.readUInt16BE(i + 2);
|
|
220
|
+
}
|
|
221
|
+
}
|
|
222
|
+
return {};
|
|
223
|
+
}
|
|
224
|
+
// Width, height and duration from the head of an MP4 whose index comes first (what web renditions use).
|
|
225
|
+
function mp4Info(b) {
|
|
226
|
+
const out = {}; let i = -1; let area = 0;
|
|
227
|
+
while ((i = b.indexOf('tkhd', i + 1)) > 3) { // one per track; the video track is the one with a size
|
|
228
|
+
const end = i - 4 + b.readUInt32BE(i - 4);
|
|
229
|
+
if (end <= b.length && end - (i - 4) >= 84) { const w = b.readUInt32BE(end - 8) >>> 16; const h = b.readUInt32BE(end - 4) >>> 16; if (w * h > area) { area = w * h; out.w = w; out.h = h; } }
|
|
230
|
+
}
|
|
231
|
+
const m = b.indexOf('mvhd');
|
|
232
|
+
if (m > 3 && m + 36 <= b.length) {
|
|
233
|
+
const v1 = b[m + 4] === 1; const scale = b.readUInt32BE(m + (v1 ? 24 : 16)); const d = v1 ? Number(b.readBigUInt64BE(m + 28)) : b.readUInt32BE(m + 20);
|
|
234
|
+
if (scale && d) out.duration = d / scale;
|
|
235
|
+
}
|
|
236
|
+
return out;
|
|
237
|
+
}
|
|
238
|
+
const clock = (s) => { if (!s && s !== 0) return null; const t = Math.round(s); return `${Math.floor(t / 60)}:${String(t % 60).padStart(2, '0')}`; };
|
|
239
|
+
// The Art Institute's image server turns away scripts that do not name themselves in this header.
|
|
240
|
+
const imageHeaders = (url) => ({ 'User-Agent': UA, Accept: 'image/*', ...(/^https?:\/\/[^/]*artic\.edu\//.test(url) ? AIC_HEADERS : {}) });
|
|
241
|
+
// Does the file load, is it something a browser shows, and how large is it? Reads only the first
|
|
242
|
+
// 128 KB of an image, or 512 KB of a video (enough for an MP4's size and duration).
|
|
243
|
+
async function probe(url, ms = 15000, video = false, again = true) {
|
|
244
|
+
const ac = new AbortController(); const timer = setTimeout(() => ac.abort(), ms);
|
|
245
|
+
const want = video ? 524288 : 131072;
|
|
246
|
+
try {
|
|
247
|
+
const headers = video ? { 'User-Agent': UA, Accept: 'video/*' } : imageHeaders(url);
|
|
248
|
+
const r = await fetch(url, { headers: { ...headers, Range: `bytes=0-${want - 1}` }, signal: ac.signal, redirect: 'follow' });
|
|
249
|
+
if (!r.ok) {
|
|
250
|
+
r.body?.cancel().catch(() => {});
|
|
251
|
+
if (r.status >= 500 && again) { clearTimeout(timer); await sleep(1000); return probe(url, ms, video, false); } // file servers stumble now and then
|
|
252
|
+
return { ok: false, why: `HTTP ${r.status}` };
|
|
253
|
+
}
|
|
254
|
+
const type = (r.headers.get('content-type') || '').split(';')[0].trim().toLowerCase();
|
|
255
|
+
if (!(video ? VIDEO_TYPES : WEB_TYPES)[type]) { r.body?.cancel().catch(() => {}); return { ok: false, why: type ? `not a web ${video ? 'video' : 'image'} (${type})` : 'no content type' }; }
|
|
256
|
+
const reader = r.body.getReader(); const chunks = []; let n = 0;
|
|
257
|
+
while (n < want) { const { done, value } = await reader.read(); if (done) break; chunks.push(value); n += value.length; }
|
|
258
|
+
reader.cancel().catch(() => {});
|
|
259
|
+
const bytes = r.status === 206 ? Number((r.headers.get('content-range') || '').split('/')[1]) || null : Number(r.headers.get('content-length')) || null;
|
|
260
|
+
const head = Buffer.concat(chunks);
|
|
261
|
+
return { ok: true, type, bytes, ...(video ? (type === 'video/mp4' ? mp4Info(head) : {}) : dims(head)) };
|
|
262
|
+
} catch (e) {
|
|
263
|
+
return { ok: false, why: e.name === 'AbortError' ? 'timed out' : e.cause?.code || e.message };
|
|
264
|
+
} finally { clearTimeout(timer); }
|
|
265
|
+
}
|
|
266
|
+
|
|
267
|
+
// ---------- licences and credits ----------
|
|
268
|
+
function licenceLabel(it) {
|
|
269
|
+
if (it.licenseName) return it.licenseName; // set where the plain code would mislead: museums, informal statements, unusual licences
|
|
270
|
+
if (it.license === 'cc0') return 'CC0';
|
|
271
|
+
if (it.license === 'pd') return 'public domain';
|
|
272
|
+
if (it.license === 'other') return 'licence on the source page';
|
|
273
|
+
return `CC ${it.license.toUpperCase()}${it.licenseVersion ? ' ' + it.licenseVersion : ''}`;
|
|
274
|
+
}
|
|
275
|
+
function licenceUrl(it) {
|
|
276
|
+
if (it.licenseUrl) return String(it.licenseUrl).replace(/^http:/, 'https:');
|
|
277
|
+
if (it.license === 'cc0') return 'https://creativecommons.org/publicdomain/zero/1.0/';
|
|
278
|
+
if (it.license === 'pd') return 'https://creativecommons.org/publicdomain/mark/1.0/';
|
|
279
|
+
if (it.license.startsWith('by')) return `https://creativecommons.org/licenses/${it.license}/${it.licenseVersion || '4.0'}/`;
|
|
280
|
+
return null;
|
|
281
|
+
}
|
|
282
|
+
const creditRequired = (it) => it.license !== 'cc0' && it.license !== 'pd';
|
|
283
|
+
function licenceNotes(it) {
|
|
284
|
+
const n = [];
|
|
285
|
+
if (/-sa$/.test(it.license)) n.push('Share-alike: an altered version (recoloured, combined with other pictures) must be shared under the same licence.');
|
|
286
|
+
if (/-nc/.test(it.license)) n.push('Non-commercial only: not for a business, a shop or a client site.');
|
|
287
|
+
if (/-nd$/.test(it.license)) n.push('No derivatives: show it whole and unaltered, no crops or treatments.');
|
|
288
|
+
if (it.license === 'other') n.push('A licence outside the usual set: read the source page before using it.');
|
|
289
|
+
return n;
|
|
290
|
+
}
|
|
291
|
+
function credit(it) {
|
|
292
|
+
const by = it.creator && !/^(unknown|anonymous)\b/i.test(it.creator) ? ` by ${short(it.creator, 60)}` : '';
|
|
293
|
+
const date = (it.source === 'aic' || it.source === 'met') && it.date ? `, ${short(it.date, 30)}` : '';
|
|
294
|
+
return `"${short(it.title, 90)}"${by}${date}, ${licenceLabel(it)}, via ${it.via}`;
|
|
295
|
+
}
|
|
296
|
+
function base(source, id) {
|
|
297
|
+
return { ref: `${source}:${id}`, source, media: 'image', via: SOURCES[source].name, title: '', creator: '', creatorUrl: null, date: null, kind: null, about: null, license: 'other', licenseVersion: null, licenseName: null, licenseUrl: null, page: null, image: null, original: null, smaller: null, width: null, height: null, imageWidth: null, imageHeight: null, bytes: null, estimated: false, duration: null, poster: null, alt: null, notes: [], hint: null };
|
|
298
|
+
}
|
|
299
|
+
// Licence code and version from a licence URL (Creative Commons licences and marks).
|
|
300
|
+
function licenceFromUrl(u) {
|
|
301
|
+
const s = String(u || '').toLowerCase();
|
|
302
|
+
if (/publicdomain\/zero/.test(s)) return { code: 'cc0', version: '1.0' };
|
|
303
|
+
if (/publicdomain\/mark|publicdomain\/?$|\/publicdomain\b/.test(s)) return { code: 'pd', version: null };
|
|
304
|
+
const m = s.match(/creativecommons\.org\/licenses\/(by(?:-nc)?(?:-sa|-nd)?)\/(\d\.\d)/);
|
|
305
|
+
return m ? { code: m[1], version: m[2] } : { code: 'other', version: null };
|
|
306
|
+
}
|
|
307
|
+
// Of the renditions a source offers, the largest whose short side is at most `res` (720p by default),
|
|
308
|
+
// in a format every browser plays. If all are larger, the smallest one.
|
|
309
|
+
function pickRendition(list, res) {
|
|
310
|
+
const playable = list.filter((r) => r.url && VIDEO_TYPES[r.type]);
|
|
311
|
+
const side = (r) => (r.w && r.h ? Math.min(r.w, r.h) : Infinity);
|
|
312
|
+
const fit = playable.filter((r) => side(r) <= res).sort((a, b) => side(b) - side(a) || (a.bytes ?? 1e15) - (b.bytes ?? 1e15));
|
|
313
|
+
return fit[0] || playable.sort((a, b) => side(a) - side(b) || (a.bytes ?? 1e15) - (b.bytes ?? 1e15))[0] || null;
|
|
314
|
+
}
|
|
315
|
+
function useRendition(it, pick, largest) {
|
|
316
|
+
Object.assign(it, { media: 'video', image: pick.url, mime: pick.type, imageWidth: pick.w || null, imageHeight: pick.h || null, bytes: pick.bytes || null, estimated: Boolean(pick.estimated), duration: it.duration || pick.duration || null });
|
|
317
|
+
if (largest) { it.width = largest.w || it.width; it.height = largest.h || it.height; }
|
|
318
|
+
return it;
|
|
319
|
+
}
|
|
320
|
+
|
|
321
|
+
// ---------- Openverse ----------
|
|
322
|
+
async function searchOpenverse(q, opts) {
|
|
323
|
+
const lic = opts.licences.filter((l) => l !== 'other').map((l) => (l === 'pd' ? 'pdm' : l));
|
|
324
|
+
if (!lic.length) return { total: 0, items: [], note: 'Openverse filters by Creative Commons licences and marks only.' };
|
|
325
|
+
const call = (large) => {
|
|
326
|
+
const p = new URLSearchParams({ q, license: lic.join(','), page_size: '20' });
|
|
327
|
+
if (opts.orientation) p.set('aspect_ratio', { landscape: 'wide', portrait: 'tall', square: 'square' }[opts.orientation]);
|
|
328
|
+
if (large) p.set('size', 'large');
|
|
329
|
+
return getJson(`https://api.openverse.org/v1/images/?${p}`, 30000);
|
|
330
|
+
};
|
|
331
|
+
// Its default order is led by 1024px Flickr files. Ask for the large ones first, and fill up from the rest only if needed.
|
|
332
|
+
const first = await call(true);
|
|
333
|
+
let rows = first.results || []; let total = first.result_count ?? null;
|
|
334
|
+
if (rows.filter((r) => !r.width || r.width >= opts.minWidth).length < opts.limit + 2) {
|
|
335
|
+
const rest = await call(false).catch(() => null);
|
|
336
|
+
if (rest) { total = rest.result_count ?? total; rows = [...rows, ...(rest.results || []).filter((r) => !rows.some((x) => x.id === r.id))]; }
|
|
337
|
+
}
|
|
338
|
+
const items = rows.map(fromOpenverse).filter((it) => it.image);
|
|
339
|
+
await wikimediaSizes(items, opts.width);
|
|
340
|
+
return { total, items };
|
|
341
|
+
}
|
|
342
|
+
function fromOpenverse(r) {
|
|
343
|
+
const it = base('openverse', r.id);
|
|
344
|
+
const via = r.source || r.provider;
|
|
345
|
+
Object.assign(it, {
|
|
346
|
+
via: PROVIDERS[via] || (String(via || '').startsWith('smithsonian') ? 'Smithsonian' : via ? String(via).replace(/_/g, ' ') : 'Openverse'),
|
|
347
|
+
title: text(r.title).replace(/^File:/, '').replace(/\.(jpe?g|png|tiff?|gif|webp)$/i, '') || 'Untitled', creator: text(r.creator), creatorUrl: r.creator_url || null,
|
|
348
|
+
license: r.license === 'pdm' ? 'pd' : String(r.license || 'other'), licenseVersion: r.license_version || null, licenseUrl: r.license_url || null,
|
|
349
|
+
page: r.foreign_landing_url || r.detail_url, image: r.url, original: r.url,
|
|
350
|
+
width: r.width || null, height: r.height || null, imageWidth: r.width || null, imageHeight: r.height || null,
|
|
351
|
+
});
|
|
352
|
+
if (!LICENCES.includes(it.license)) it.license = 'other';
|
|
353
|
+
return it;
|
|
354
|
+
}
|
|
355
|
+
// Openverse points at Wikimedia originals, which can be 10 MB or a TIFF. Swap in a web-sized rendition.
|
|
356
|
+
async function wikimediaSizes(items, width) {
|
|
357
|
+
const wm = items.filter((it) => /^https?:\/\/upload\.wikimedia\.org\/wikipedia\/commons\/[0-9a-f]\/[0-9a-f]{2}\//.test(it.original || ''));
|
|
358
|
+
if (!wm.length) return;
|
|
359
|
+
const title = (it) => { let n = noQuery(it.original).split('/').pop(); try { n = decodeURIComponent(n); } catch {} return 'File:' + n.replace(/_/g, ' '); };
|
|
360
|
+
const j = await getJson(commonsApi({ titles: wm.map(title).join('|'), prop: 'imageinfo', iiprop: 'url|size|mime', iiurlwidth: String(width) })).catch(() => null);
|
|
361
|
+
if (!j?.query) return;
|
|
362
|
+
const normal = new Map((j.query.normalized || []).map((n) => [n.from, n.to]));
|
|
363
|
+
const info = new Map((j.query.pages || []).map((pg) => [pg.title, pg.imageinfo?.[0]]));
|
|
364
|
+
for (const it of wm) { const t = title(it); const ii = info.get(normal.get(t) || t); if (ii) applyThumb(it, ii); }
|
|
365
|
+
}
|
|
366
|
+
|
|
367
|
+
// ---------- Wikimedia Commons ----------
|
|
368
|
+
function commonsApi(params) { return `https://commons.wikimedia.org/w/api.php?${new URLSearchParams({ action: 'query', format: 'json', formatversion: '2', origin: '*', ...params })}`; }
|
|
369
|
+
function commonsInfo(width) {
|
|
370
|
+
return { prop: 'imageinfo', iiprop: 'url|size|mime|extmetadata', iiurlwidth: String(width), iiextmetadatalanguage: 'en',
|
|
371
|
+
iiextmetadatafilter: 'ObjectName|Artist|LicenseShortName|License|LicenseUrl|Restrictions|DateTimeOriginal|ImageDescription' };
|
|
372
|
+
}
|
|
373
|
+
async function searchCommons(q, opts) {
|
|
374
|
+
let s = `${q} filetype:bitmap`;
|
|
375
|
+
if (opts.minWidth > 1) s += ` filew:>${opts.minWidth - 1}`;
|
|
376
|
+
const j = await getJson(commonsApi({ generator: 'search', gsrnamespace: '6', gsrsearch: s, gsrlimit: '30', ...commonsInfo(opts.width) }));
|
|
377
|
+
if (j.error) throw new Error(j.error.info || 'the search failed');
|
|
378
|
+
const pages = (j.query?.pages || []).sort((a, b) => (a.index ?? 0) - (b.index ?? 0));
|
|
379
|
+
return { total: null, items: pages.map(fromCommons).filter(Boolean) };
|
|
380
|
+
}
|
|
381
|
+
function fromCommons(pg) {
|
|
382
|
+
const ii = pg?.imageinfo?.[0]; if (!ii) return null;
|
|
383
|
+
const m = (k) => text(ii.extmetadata?.[k]?.value);
|
|
384
|
+
const lic = commonsLicence(m('License'), m('LicenseShortName'));
|
|
385
|
+
const it = base('commons', pg.pageid);
|
|
386
|
+
Object.assign(it, {
|
|
387
|
+
title: m('ObjectName').replace(/\s*\b(?:title|label) QS:.*$/s, '') || pg.title.replace(/^File:/, '').replace(/\.[a-z0-9]+$/i, ''), creator: m('Artist'),
|
|
388
|
+
date: m('DateTimeOriginal').replace(/\s*\bdate QS:.*$/s, '').replace(/^(\d{4}-\d{2}-\d{2})[ T]\d{2}:\d{2}.*$/, '$1') || null, about: m('ImageDescription') || null,
|
|
389
|
+
license: lic.code, licenseVersion: lic.version, licenseName: lic.name, licenseUrl: m('LicenseUrl') || null, page: ii.descriptionurl,
|
|
390
|
+
});
|
|
391
|
+
applyThumb(it, ii);
|
|
392
|
+
if (lic.informal) it.notes.push('Not a formal licence: the archive that published it states it knows of no copyright restrictions.');
|
|
393
|
+
if (it.about && it.about.toLowerCase().startsWith(it.title.toLowerCase()) && it.about.length < it.title.length + 12) it.about = null;
|
|
394
|
+
const restrictions = m('Restrictions');
|
|
395
|
+
if (restrictions) it.notes.push(`Restrictions named on the source page (${restrictions}): personality rights, a trademark or similar. Read it before use.`);
|
|
396
|
+
if (/^pd-us/i.test(m('License'))) it.notes.push('Public domain in the United States; check the source page before using it elsewhere.');
|
|
397
|
+
return it;
|
|
398
|
+
}
|
|
399
|
+
function commonsLicence(license, shortName) {
|
|
400
|
+
const l = String(license || '').toLowerCase(); const s = String(shortName || '').toLowerCase();
|
|
401
|
+
const version = (shortName.match(/\b(\d\.\d)\b/) || l.match(/(\d\.\d)/) || [])[1] || null;
|
|
402
|
+
if (/^cc0|^cc-zero/.test(l) || /^cc0/.test(s)) return { code: 'cc0', version: '1.0' };
|
|
403
|
+
if (/^pd|public domain/.test(l) || /public domain/.test(s)) return { code: 'pd', version: null };
|
|
404
|
+
// Flickr Commons and similar archives: no formal licence, the institution states it knows of no copyright
|
|
405
|
+
if (/^no (known copyright )?restrictions/.test(s)) return { code: 'pd', version: null, name: 'no known copyright restrictions', informal: true };
|
|
406
|
+
const cc = l.match(/^cc-(by(?:-nc)?(?:-sa|-nd)?)(?:-|$)/) || s.match(/^cc (by(?:-nc)?(?:-sa|-nd)?)\b/);
|
|
407
|
+
if (cc) return { code: cc[1], version };
|
|
408
|
+
return { code: 'other', version: null, name: shortName || license || null };
|
|
409
|
+
}
|
|
410
|
+
// Commons serves thumbnails only at standard widths (1280, 1920, 3840...); the API picks the step.
|
|
411
|
+
function applyThumb(it, ii) {
|
|
412
|
+
it.width = ii.width || it.width; it.height = ii.height || it.height; it.original = noQuery(ii.url) || it.original;
|
|
413
|
+
const tw = Number(String(ii.thumburl || '').split('?')[0].match(/(\d+)px-[^/]*$/)?.[1]) || 0;
|
|
414
|
+
if (ii.thumburl && tw && ii.width && (tw < ii.width || (ii.mime && !WEB_TYPES[ii.mime]))) {
|
|
415
|
+
it.image = noQuery(ii.thumburl); it.imageWidth = Math.min(tw, ii.width); it.imageHeight = ii.height ? Math.round((ii.height * it.imageWidth) / ii.width) : null;
|
|
416
|
+
} else { it.image = it.original; it.imageWidth = ii.width || null; it.imageHeight = ii.height || null; }
|
|
417
|
+
}
|
|
418
|
+
|
|
419
|
+
// ---------- Art Institute of Chicago ----------
|
|
420
|
+
async function searchAic(q, opts) {
|
|
421
|
+
const wanted = queryWords(q);
|
|
422
|
+
const run = (match) => getJson(`https://api.artic.edu/api/v1/artworks/search?params=${encodeURIComponent(JSON.stringify({
|
|
423
|
+
query: { bool: {
|
|
424
|
+
// the long description field is left out on purpose: it matches almost anything
|
|
425
|
+
must: [{ multi_match: { query: wanted.join(' ') || q, type: 'cross_fields', fields: ['title^3', 'subject_titles^2', 'term_titles^2', 'classification_titles', 'medium_display', 'artist_title', 'place_of_origin', 'date_display'], ...match } }],
|
|
426
|
+
filter: [{ term: { is_public_domain: true } }, { exists: { field: 'image_id' } }],
|
|
427
|
+
} },
|
|
428
|
+
fields: AIC_FIELDS, limit: 30,
|
|
429
|
+
}))}`, 20000, AIC_HEADERS);
|
|
430
|
+
// every word must match; with three words or more, one may be missing. Looser than that returns noise.
|
|
431
|
+
let j = await run({ operator: 'and' });
|
|
432
|
+
if (!(j.data || []).length && wanted.length >= 3) j = await run({ operator: 'or', minimum_should_match: wanted.length - 1 });
|
|
433
|
+
const iiif = j.config?.iiif_url || 'https://www.artic.edu/iiif/2';
|
|
434
|
+
return { total: j.pagination?.total ?? null, items: (j.data || []).map((d) => fromAic(d, iiif, opts.width)).filter(Boolean) };
|
|
435
|
+
}
|
|
436
|
+
function fromAic(d, iiif, width) {
|
|
437
|
+
if (!d?.image_id || d.is_public_domain === false) return null;
|
|
438
|
+
const it = base('aic', d.id);
|
|
439
|
+
const W = d.thumbnail?.width || null; const H = d.thumbnail?.height || null;
|
|
440
|
+
const cap = W ? Math.min(W, 3000) : 3000; const w = Math.min(width, cap);
|
|
441
|
+
Object.assign(it, {
|
|
442
|
+
title: text(d.title) || 'Untitled', creator: text(d.artist_title) || text(String(d.artist_display || '').split('\n')[0]),
|
|
443
|
+
date: text(d.date_display) || null, kind: [d.artwork_type_title, d.medium_display].filter(Boolean).map(text).join(', ') || null,
|
|
444
|
+
license: 'cc0', licenseVersion: '1.0', licenseName: 'public domain (CC0)', licenseUrl: 'https://creativecommons.org/publicdomain/zero/1.0/',
|
|
445
|
+
page: `https://www.artic.edu/artworks/${d.id}`, image: `${iiif}/${d.image_id}/full/${w},/0/default.jpg`, original: `${iiif}/${d.image_id}/full/${cap},/0/default.jpg`,
|
|
446
|
+
width: W && H ? cap : null, height: W && H ? Math.round((H * cap) / W) : null, imageWidth: W && H ? w : null, imageHeight: W && H ? Math.round((H * w) / W) : null,
|
|
447
|
+
alt: text(d.thumbnail?.alt_text) || null,
|
|
448
|
+
});
|
|
449
|
+
it.hint = 'Download it with --get: the museum\'s image server answers browsers and this script, and refuses plain curl or wget.';
|
|
450
|
+
return it;
|
|
451
|
+
}
|
|
452
|
+
|
|
453
|
+
// ---------- The Met ----------
|
|
454
|
+
async function searchMet(q, opts) {
|
|
455
|
+
const s = await getJson(`https://collectionapi.metmuseum.org/public/collection/v1.1/search?${new URLSearchParams({ q, hasImages: 'true', limit: String(Math.min(40, opts.limit * 4 + 8)), offset: '0' })}`);
|
|
456
|
+
const ids = (s.objectIDs || []).slice(0, opts.limit * 4 + 8);
|
|
457
|
+
const objects = await mapLimit(ids, 8, (id) => getJson(`https://collectionapi.metmuseum.org/public/collection/v1/objects/${id}`, 15000).catch(() => null));
|
|
458
|
+
// The museum's search reads every field of a record, so much of what it returns is off the subject.
|
|
459
|
+
// Keep the objects whose own name, material, maker, place or tags carry the words of the query.
|
|
460
|
+
const wanted = queryWords(q);
|
|
461
|
+
const about = (x) => [x.title, x.objectName, x.medium, x.classification, x.culture, x.period, x.artistDisplayName, x.department, x.city, x.country, x.region, x.objectDate, ...(x.tags || []).map((t) => t.term)].filter(Boolean).join(' ');
|
|
462
|
+
return { total: s.total ?? null, items: objects.filter((x) => x && onSubject(wanted, about(x))).map(fromMet).filter(Boolean) };
|
|
463
|
+
}
|
|
464
|
+
function fromMet(x) {
|
|
465
|
+
if (!x || !x.isPublicDomain || !x.primaryImage) return null;
|
|
466
|
+
const it = base('met', x.objectID);
|
|
467
|
+
const fix = (u) => (u ? encodeURI(u).replace(/%25([0-9a-f]{2})/gi, '%$1') : null);
|
|
468
|
+
Object.assign(it, {
|
|
469
|
+
title: text(x.title) || text(x.objectName) || 'Untitled', creator: text(x.artistDisplayName) || (x.culture ? `${text(x.culture)} maker` : ''),
|
|
470
|
+
date: text(x.objectDate) || null, kind: [x.objectName || x.classification, x.medium].filter(Boolean).map(text).join(', ') || null,
|
|
471
|
+
license: 'cc0', licenseVersion: '1.0', licenseName: 'public domain (CC0)', licenseUrl: 'https://creativecommons.org/publicdomain/zero/1.0/',
|
|
472
|
+
page: x.objectURL || `https://www.metmuseum.org/art/collection/search/${x.objectID}`,
|
|
473
|
+
image: fix(x.primaryImage), original: fix(x.primaryImage), smaller: x.primaryImageSmall && x.primaryImageSmall !== x.primaryImage ? fix(x.primaryImageSmall) : null,
|
|
474
|
+
});
|
|
475
|
+
return it;
|
|
476
|
+
}
|
|
477
|
+
|
|
478
|
+
// ---------- video: Wikimedia Commons ----------
|
|
479
|
+
async function searchCommonsVideo(q, opts) {
|
|
480
|
+
const j = await getJson(commonsApi({ generator: 'search', gsrnamespace: '6', gsrsearch: `${q} filetype:video`, gsrlimit: '30', ...commonsInfo(1280), prop: 'imageinfo|videoinfo', viprop: 'derivatives' }));
|
|
481
|
+
if (j.error) throw new Error(j.error.info || 'the search failed');
|
|
482
|
+
const pages = (j.query?.pages || []).sort((a, b) => (a.index ?? 0) - (b.index ?? 0));
|
|
483
|
+
return { total: null, items: pages.map((pg) => fromCommonsVideo(pg, opts)).filter(Boolean) };
|
|
484
|
+
}
|
|
485
|
+
function fromCommonsVideo(pg, opts) {
|
|
486
|
+
const it = fromCommons(pg); const ii = pg?.imageinfo?.[0];
|
|
487
|
+
if (!it || !ii) return null;
|
|
488
|
+
it.poster = it.image !== it.original ? it.image : null; // for a video file, the thumbnail is a still frame
|
|
489
|
+
it.duration = Number(ii.duration) || null;
|
|
490
|
+
// the original plus the transcodes Commons makes of every video (240p to 1080p webm); sizes from their bit rates
|
|
491
|
+
const renditions = (pg.videoinfo?.[0]?.derivatives || []).map((d) => {
|
|
492
|
+
const url = noQuery(d.src); const original = !/\/transcoded\//.test(url);
|
|
493
|
+
return { url, type: String(d.type || '').split(';')[0].trim(), w: Number(d.width) || null, h: Number(d.height) || null,
|
|
494
|
+
bytes: original ? Number(ii.size) || null : d.bandwidth && it.duration ? Math.round((d.bandwidth * it.duration) / 8) : null, estimated: !original };
|
|
495
|
+
});
|
|
496
|
+
const pick = pickRendition(renditions, opts.res);
|
|
497
|
+
return pick ? useRendition(it, pick, { w: ii.width, h: ii.height }) : null;
|
|
498
|
+
}
|
|
499
|
+
|
|
500
|
+
// ---------- video: Internet Archive ----------
|
|
501
|
+
const first = (v) => (Array.isArray(v) ? v[0] : v);
|
|
502
|
+
const seconds = (s) => {
|
|
503
|
+
if (s === undefined || s === null || s === '') return null;
|
|
504
|
+
if (/^\d+(\.\d+)?$/.test(String(s))) return Number(s);
|
|
505
|
+
const p = String(s).split(':').map(Number); return p.length > 1 && p.every(Number.isFinite) ? p.reduce((a, x) => a * 60 + x, 0) : null;
|
|
506
|
+
};
|
|
507
|
+
// Only the Prelinger Archives: curated industrial, educational, advertising and home films, mostly in the
|
|
508
|
+
// public domain. The rest of the archive's films are uploads by anyone, and their licence claims go
|
|
509
|
+
// unchecked (an upload called "... all rights reserved" can carry a public-domain mark), so they are left out.
|
|
510
|
+
const ARCHIVE_LICENCE = { cc0: '*publicdomain\\/zero*', pd: '*publicdomain*', by: '*licenses\\/by\\/*', 'by-sa': '*licenses\\/by-sa\\/*', 'by-nc': '*licenses\\/by-nc\\/*', 'by-nd': '*licenses\\/by-nd\\/*', 'by-nc-sa': '*licenses\\/by-nc-sa\\/*', 'by-nc-nd': '*licenses\\/by-nc-nd\\/*' };
|
|
511
|
+
async function searchArchive(q, opts) {
|
|
512
|
+
const words = queryWords(q);
|
|
513
|
+
const licences = opts.licences.includes('other') ? '' : ` AND licenseurl:(${[...new Set(opts.licences.map((l) => ARCHIVE_LICENCE[l]))].join(' OR ')})`;
|
|
514
|
+
const p = new URLSearchParams({ q: `(${(words.length ? words : [q]).join(' AND ')}) AND mediatype:(movies) AND collection:(prelinger)${licences}`, rows: '40', output: 'json' });
|
|
515
|
+
for (const f of ['identifier', 'licenseurl']) p.append('fl[]', f);
|
|
516
|
+
const j = await getJson(`https://archive.org/advancedsearch.php?${p}`, 30000);
|
|
517
|
+
if (j.error) throw new Error(`its search is failing right now (${short(text(j.error), 80)}); try again later`);
|
|
518
|
+
const docs = j.response?.docs || [];
|
|
519
|
+
const allowed = docs.filter((d) => opts.licences.includes(licenceFromUrl(first(d.licenseurl)).code));
|
|
520
|
+
const metas = await mapLimit(allowed.slice(0, opts.limit * 2 + 4), 6, (d) => getJson(`https://archive.org/metadata/${encodeURIComponent(d.identifier)}`, 20000).catch(() => null));
|
|
521
|
+
return { total: j.response?.numFound ?? null, skippedLicence: docs.length - allowed.length, items: metas.map((m) => fromArchive(m, opts)).filter(Boolean) };
|
|
522
|
+
}
|
|
523
|
+
function fromArchive(m, opts) {
|
|
524
|
+
const md = m?.metadata; if (!md?.identifier || !Array.isArray(m.files)) return null;
|
|
525
|
+
const id = md.identifier;
|
|
526
|
+
const fileUrl = (name) => `https://archive.org/download/${encodeURIComponent(id)}/${String(name).split('/').map(encodeURIComponent).join('/')}`;
|
|
527
|
+
const isVideo = (f) => /\.(mp4|m4v|webm|mpe?g|avi|mov|ogv|mkv|wmv|flv)$/i.test(f.name || '');
|
|
528
|
+
// an item can hold several films: keep the first original and the web copies made from it
|
|
529
|
+
const originals = m.files.filter((f) => f.source === 'original' && isVideo(f));
|
|
530
|
+
const stem = originals[0] ? originals[0].name.replace(/\.[^.]+$/, '') : null;
|
|
531
|
+
const family = m.files.filter((f) => isVideo(f) && (!stem || f.name.startsWith(stem)));
|
|
532
|
+
const renditions = family.filter((f) => /\.(mp4|webm)$/i.test(f.name)).map((f) => ({ url: fileUrl(f.name), type: /\.webm$/i.test(f.name) ? 'video/webm' : 'video/mp4', w: Number(f.width) || null, h: Number(f.height) || null, bytes: Number(f.size) || null, duration: seconds(f.length) }));
|
|
533
|
+
const pick = pickRendition(renditions, opts.res); if (!pick) return null;
|
|
534
|
+
const largest = family.reduce((a, f) => ((Number(f.width) || 0) > (a.w || 0) ? { w: Number(f.width), h: Number(f.height) } : a), { w: pick.w, h: pick.h });
|
|
535
|
+
const lic = licenceFromUrl(first(md.licenseurl));
|
|
536
|
+
const it = base('archive', id);
|
|
537
|
+
Object.assign(it, {
|
|
538
|
+
title: text(first(md.title)) || id, creator: text(first(md.creator)), date: text(first(md.date) || first(md.year)) || null, about: text(first(md.description)) || null,
|
|
539
|
+
license: lic.code, licenseVersion: lic.version, licenseUrl: first(md.licenseurl) || null, page: `https://archive.org/details/${id}`,
|
|
540
|
+
original: fileUrl((originals[0] || family[0]).name), duration: pick.duration || seconds(first(md.runtime)),
|
|
541
|
+
poster: null, // the archive's own stills are about 160 pixels wide: too small to stand in for a clip
|
|
542
|
+
});
|
|
543
|
+
// the same film is often stored in several formats; different names mean different films
|
|
544
|
+
const films = new Set(originals.map((f) => f.name.split('/').pop().split('.')[0])).size;
|
|
545
|
+
if (films > 1) it.notes.push(`This item holds ${films} films; the first is shown. The page lists the others.`);
|
|
546
|
+
return useRendition(it, pick, largest);
|
|
547
|
+
}
|
|
548
|
+
|
|
549
|
+
// ---------- video: NASA Image and Video Library ----------
|
|
550
|
+
const nasaUrl = (u) => (u ? encodeURI(String(u).replace(/^http:/, 'https:')).replace(/%25([0-9a-f]{2})/gi, '%$1') : null);
|
|
551
|
+
async function searchNasa(q, opts) {
|
|
552
|
+
const j = await getJson(`https://images-api.nasa.gov/search?${new URLSearchParams({ q, media_type: 'video', page_size: String(Math.min(100, opts.limit * 3 + 6)) })}`);
|
|
553
|
+
// its search matches any word anywhere; keep the clips whose own title, description or keywords carry the query
|
|
554
|
+
const wanted = queryWords(q);
|
|
555
|
+
const rows = (j.collection?.items || []).filter((x) => { const d = x.data?.[0] || {}; return onSubject(wanted, [d.title, d.description, ...(d.keywords || [])].join(' ')); }).slice(0, opts.limit * 2 + 4);
|
|
556
|
+
const items = await mapLimit(rows, 6, async (x) => fromNasa(x, await getJson(nasaUrl(x.href), 15000).catch(() => null), opts));
|
|
557
|
+
return { total: j.collection?.metadata?.total_hits ?? null, items: items.filter(Boolean) };
|
|
558
|
+
}
|
|
559
|
+
function fromNasa(x, files, opts) {
|
|
560
|
+
const d = x?.data?.[0]; if (!d?.nasa_id || !Array.isArray(files)) return null;
|
|
561
|
+
const find = (suffix) => files.find((f) => String(f).endsWith(suffix));
|
|
562
|
+
// NASA makes the same renditions of every video; these are their 16:9 sizes, and the check reads the real ones
|
|
563
|
+
const renditions = [['~large.mp4', 1920, 1080], ['~medium.mp4', 1280, 720], ['~small.mp4', 640, 360], ['~mobile.mp4', 320, 180]]
|
|
564
|
+
.filter(([s]) => find(s)).map(([s, w, h]) => ({ url: nasaUrl(find(s)), type: 'video/mp4', w, h }));
|
|
565
|
+
const pick = pickRendition(renditions, opts.res); if (!pick) return null;
|
|
566
|
+
const it = base('nasa', encodeURIComponent(d.nasa_id));
|
|
567
|
+
Object.assign(it, {
|
|
568
|
+
title: text(d.title) || d.nasa_id, creator: text(d.photographer || d.secondary_creator) || `NASA${d.center ? ' ' + d.center : ''}`,
|
|
569
|
+
date: String(d.date_created || '').slice(0, 10) || null, about: text(d.description) || null,
|
|
570
|
+
license: 'pd', licenseName: 'public domain (NASA)', licenseUrl: 'https://www.nasa.gov/nasa-brand-center/images-and-media/',
|
|
571
|
+
page: `https://images.nasa.gov/details/${encodeURIComponent(d.nasa_id)}`, original: nasaUrl(find('~orig.mp4')) || pick.url,
|
|
572
|
+
poster: nasaUrl(find('~large.jpg') || find('~medium.jpg') || find('~thumb.jpg')),
|
|
573
|
+
});
|
|
574
|
+
it.notes.push('NASA footage is generally free of copyright in the US. Never use NASA logos or suggest NASA endorses the client; check the page for third-party material.');
|
|
575
|
+
return useRendition(it, pick, renditions[0]);
|
|
576
|
+
}
|
|
577
|
+
function videoNotes(it, opts) {
|
|
578
|
+
const n = [];
|
|
579
|
+
const smaller = it.imageWidth && it.imageHeight && Math.min(it.imageWidth, it.imageHeight) > 480;
|
|
580
|
+
if (it.bytes && it.bytes > opts.maxMb * 1048576) n.push(`Over ${opts.maxMb} MB (${it.estimated ? 'about ' : ''}${size(it.bytes)}): too heavy for a page, and --get refuses it unless you pass --max-mb. ${smaller ? 'Try --res 480, or a shorter clip.' : 'Choose a shorter clip.'}`);
|
|
581
|
+
if (it.duration && it.duration > 60) n.push(`Long (${clock(it.duration)}): a loop on a page wants 5 to 20 seconds. Prefer a shorter clip, or trim one with ffmpeg if it is installed.`);
|
|
582
|
+
return n;
|
|
583
|
+
}
|
|
584
|
+
|
|
585
|
+
// ---------- search ----------
|
|
586
|
+
function fits(it, opts, skipped) {
|
|
587
|
+
const W = it.width || it.imageWidth; const H = it.height || it.imageHeight;
|
|
588
|
+
if (W && W < opts.minWidth) { skipped.small++; return false; }
|
|
589
|
+
if (opts.orientation && W && H && shape(W, H) !== opts.orientation) { skipped.shape++; return false; }
|
|
590
|
+
return true;
|
|
591
|
+
}
|
|
592
|
+
async function search(query, opts) {
|
|
593
|
+
const runners = opts.video ? { commons: searchCommonsVideo, archive: searchArchive, nasa: searchNasa } : { openverse: searchOpenverse, commons: searchCommons, aic: searchAic, met: searchMet };
|
|
594
|
+
// a slow service never holds the whole search: each source gets 45 s, the check 30 s
|
|
595
|
+
const deadline = (ms, message) => new Promise((_, reject) => setTimeout(() => reject(new Error(message)), ms).unref());
|
|
596
|
+
const results = await Promise.all(opts.sources.map(async (key) => {
|
|
597
|
+
try { return { key, ...(await Promise.race([runners[key](query, opts), deadline(45000, 'no complete answer within 45 s; try again or search another --source')])) }; }
|
|
598
|
+
catch (e) { return { key, error: e.message, items: [] }; }
|
|
599
|
+
}));
|
|
600
|
+
// short clips first, then longer ones, then anything over the size cap; the source's own order within each
|
|
601
|
+
const cap = opts.maxMb * 1048576;
|
|
602
|
+
const rank = (it) => (it.media !== 'video' ? 0 : it.bytes && it.bytes > cap ? 4 : !it.duration ? 3 : it.duration <= 30 ? 0 : it.duration <= 90 ? 1 : it.duration <= 300 ? 2 : 3);
|
|
603
|
+
const byRank = (a, b) => rank(a) - rank(b);
|
|
604
|
+
const seen = new Set();
|
|
605
|
+
for (const r of results) {
|
|
606
|
+
r.skipped = { licence: r.skippedLicence || 0, small: 0, shape: 0, dead: 0, alike: 0 };
|
|
607
|
+
const titles = new Map();
|
|
608
|
+
r.items = r.items.filter((it) => {
|
|
609
|
+
if (!opts.licences.includes(it.license)) { r.skipped.licence++; return false; }
|
|
610
|
+
const k = keyOf(it.original || it.image); if (seen.has(k)) return false; seen.add(k);
|
|
611
|
+
if (!fits(it, opts, r.skipped)) return false;
|
|
612
|
+
// one scanned book can fill a whole page of results: keep two pictures per title
|
|
613
|
+
const t = it.title.toLowerCase().replace(/[^a-z0-9]+/g, ' ').trim().slice(0, 32); const n = (titles.get(t) || 0) + 1; titles.set(t, n);
|
|
614
|
+
if (t && n > 2) { r.skipped.alike++; return false; }
|
|
615
|
+
return true;
|
|
616
|
+
}).sort(byRank).slice(0, opts.limit + 4); // a few spares for files that turn out not to load
|
|
617
|
+
}
|
|
618
|
+
if (opts.check) {
|
|
619
|
+
await Promise.race([mapLimit(results.flatMap((r) => r.items), 8, async (it) => {
|
|
620
|
+
const video = it.media === 'video';
|
|
621
|
+
const p = await probe(it.image, video ? 15000 : 12000, video); it.check = p;
|
|
622
|
+
if (p.ok && p.w && p.h) {
|
|
623
|
+
it.imageWidth = p.w; it.imageHeight = p.h;
|
|
624
|
+
if (!it.width || it.image === it.original) { it.width = p.w; it.height = p.h; }
|
|
625
|
+
}
|
|
626
|
+
if (p.ok && p.bytes) { it.bytes = p.bytes; it.estimated = false; }
|
|
627
|
+
if (p.ok && p.duration) it.duration = p.duration;
|
|
628
|
+
}), new Promise((r) => setTimeout(r, 30000).unref())]);
|
|
629
|
+
for (const r of results) r.items = r.items.filter((it) => (it.check?.ok ? fits(it, opts, r.skipped) : (r.skipped.dead++, (r.skipped.why = it.check?.why || 'not checked within 30 s'), false))).sort(byRank);
|
|
630
|
+
}
|
|
631
|
+
for (const r of results) r.items = r.items.slice(0, opts.limit);
|
|
632
|
+
const shown = results.reduce((n, r) => n + r.items.length, 0);
|
|
633
|
+
|
|
634
|
+
if (args.json) {
|
|
635
|
+
console.log(JSON.stringify({ query, options: { media: opts.video ? 'video' : 'image', sources: opts.sources, licences: opts.licences, orientation: opts.orientation, minWidth: opts.minWidth, width: opts.width, res: opts.video ? opts.res : undefined, maxMb: opts.video ? opts.maxMb : undefined, limit: opts.limit, checked: opts.check },
|
|
636
|
+
sources: results.map((r) => ({ source: r.key, name: SOURCES[r.key].name, total: r.total ?? null, error: r.error || null, skipped: r.skipped,
|
|
637
|
+
items: r.items.map(({ check, ...it }) => ({ ...it, licenseLabel: licenceLabel(it), licenseUrl: licenceUrl(it), creditRequired: creditRequired(it), credit: credit(it), notes: [...it.notes, ...licenceNotes(it), ...(it.media === 'video' ? videoNotes(it, opts) : [])] })) })) }, null, 1));
|
|
638
|
+
return shown ? 0 : 1;
|
|
639
|
+
}
|
|
640
|
+
|
|
641
|
+
const filters = opts.video
|
|
642
|
+
? [`licences ${opts.licences.join(', ')}`, `at least ${opts.minWidth}px wide`, opts.orientation, `renditions up to ${opts.res}p`, `flagged over ${opts.maxMb} MB`, opts.check ? 'every file checked' : 'files not checked']
|
|
643
|
+
: [`licences ${opts.licences.join(', ')}`, `at least ${opts.minWidth}px wide`, opts.orientation, `image URLs ${opts.width}px wide where the source can size them`, opts.check ? 'every image checked' : 'images not checked'];
|
|
644
|
+
console.log(`ASSETS${opts.video ? ' · VIDEO' : ''} · "${query}" · ${filters.filter(Boolean).join(' · ')}`);
|
|
645
|
+
for (const r of results) {
|
|
646
|
+
const src = SOURCES[r.key];
|
|
647
|
+
console.log('');
|
|
648
|
+
if (r.error) { console.log(`## ${src.name}: could not search it (${r.error})`); continue; }
|
|
649
|
+
const dropped = [r.skipped.small && `${r.skipped.small} narrower than ${opts.minWidth}px`, r.skipped.shape && `${r.skipped.shape} not ${opts.orientation}`, r.skipped.licence && `${r.skipped.licence} under other licences`, r.skipped.alike && `${r.skipped.alike} more with the same title`, r.skipped.dead && `${r.skipped.dead} did not load (${r.skipped.why})`].filter(Boolean);
|
|
650
|
+
const count = r.total != null ? `${r.total} matches, ${r.items.length} shown` : `${r.items.length} shown`;
|
|
651
|
+
console.log(`## ${src.name}: ${r.items.length ? count : 'nothing usable'}${dropped.length ? ` (dropped: ${dropped.join(', ')})` : ''}`);
|
|
652
|
+
if (r.note) console.log(r.note);
|
|
653
|
+
if (!r.items.length) continue;
|
|
654
|
+
console.log(`${(opts.video && src.videoNote) || src.note}`);
|
|
655
|
+
for (const it of r.items) {
|
|
656
|
+
console.log('');
|
|
657
|
+
console.log(`${it.ref} ${short(it.title, 100)}${it.date ? ` (${short(it.date, 30)})` : ''}`);
|
|
658
|
+
if (it.kind) console.log(` what ${short(it.kind, 110)}`);
|
|
659
|
+
if (it.about) console.log(` about ${short(it.about, 150)}`);
|
|
660
|
+
const pixels = it.imageWidth && it.imageHeight ? `${it.imageWidth} × ${it.imageHeight}` : 'size unknown';
|
|
661
|
+
const largest = it.width && it.height && it.width !== it.imageWidth ? `; largest ${it.width} × ${it.height}` : '';
|
|
662
|
+
const sh = shape(it.width || it.imageWidth, it.height || it.imageHeight);
|
|
663
|
+
if (it.media === 'video') {
|
|
664
|
+
const facts = [pixels, it.duration ? clock(it.duration) : 'length unknown', it.bytes ? `${it.estimated ? 'about ' : ''}${size(it.bytes)}` : null, VIDEO_TYPES[it.mime]];
|
|
665
|
+
console.log(` file ${it.image}`);
|
|
666
|
+
console.log(` size ${facts.filter(Boolean).join(', ')}${largest}${sh ? `; ${sh}` : ''}`);
|
|
667
|
+
if (it.poster) console.log(` poster ${it.poster}`);
|
|
668
|
+
} else {
|
|
669
|
+
console.log(` image ${it.image}`);
|
|
670
|
+
console.log(` size ${pixels}${it.bytes ? `, ${size(it.bytes)}` : ''}${largest}${sh ? `; ${sh}` : ''}`);
|
|
671
|
+
if (it.smaller) console.log(` smaller ${it.smaller}`);
|
|
672
|
+
}
|
|
673
|
+
console.log(` page ${it.page}`);
|
|
674
|
+
const lu = licenceUrl(it);
|
|
675
|
+
console.log(` licence ${licenceLabel(it)}${lu ? ` (${lu})` : ''}; ${creditRequired(it) ? 'credit required' : 'no credit required, credit anyway'}`);
|
|
676
|
+
console.log(` credit ${credit(it)}`);
|
|
677
|
+
for (const n of [...(it.media === 'video' ? videoNotes(it, opts) : []), ...it.notes, ...licenceNotes(it), ...(it.hint ? [it.hint] : [])]) console.log(` note ${n}`);
|
|
678
|
+
}
|
|
679
|
+
}
|
|
680
|
+
console.log('');
|
|
681
|
+
if (results.every((r) => r.error)) {
|
|
682
|
+
console.log('No source could be reached. Check the network and try again; if this machine has no internet access, draw the');
|
|
683
|
+
console.log('pictures instead (references/imagery.md) and say in your report that the open collections could not be searched.');
|
|
684
|
+
return 1;
|
|
685
|
+
}
|
|
686
|
+
if (!shown && opts.video) {
|
|
687
|
+
console.log('Nothing usable came back. Name the action or the process ("coffee roasting", "timber framing", "river"); use fewer');
|
|
688
|
+
console.log('words; try a smaller --min-width or another --source. A page does not need video: a still picture or a drawing is fine.');
|
|
689
|
+
return 1;
|
|
690
|
+
}
|
|
691
|
+
if (!shown) {
|
|
692
|
+
console.log('Nothing usable came back. Try fewer words; the place\'s own name for things ("azulejo", not "blue tile"); the place and a');
|
|
693
|
+
console.log('medium ("ghent postcard", "ghent old map", "ghent engraving"); a decade ("rotterdam harbour 1920"); a smaller --min-width;');
|
|
694
|
+
console.log('or another --source. The museum sources match one or two words best ("fig", "harbour"). If nothing honest exists, draw it.');
|
|
695
|
+
return 1;
|
|
696
|
+
}
|
|
697
|
+
if (opts.video) console.log('Next. Watch before you choose: --get downloads the clip and a poster frame into the project and records the credit.');
|
|
698
|
+
else console.log('Next. Look before you choose: --get downloads into the project and records the credit; open the file to see it.');
|
|
699
|
+
console.log(` ${CMD} --get <ref>[,<ref>...]${o.toArg}`);
|
|
700
|
+
console.log(` ${CMD} --credits${o.toArg} the footer credit line and the list for your report`);
|
|
701
|
+
console.log(`These ${opts.video ? 'clips' : 'pictures'} show the subject's world. Never present one as the client's own product, room, dish, person or team.`);
|
|
702
|
+
return 0;
|
|
703
|
+
}
|
|
704
|
+
|
|
705
|
+
// ---------- get ----------
|
|
706
|
+
function creditsPath(opts) { return join(opts.to, 'credits.json'); }
|
|
707
|
+
function readCredits(opts) { try { const j = JSON.parse(readFileSync(creditsPath(opts), 'utf8')); return Array.isArray(j) ? j : []; } catch { return []; } }
|
|
708
|
+
// A file name in the target folder that is free, or already belongs to this same picture.
|
|
709
|
+
function targetFile(opts, stem, ext, ref, credits) {
|
|
710
|
+
let file = join(opts.to, `${stem}.${ext}`);
|
|
711
|
+
for (let k = 2; existsSync(file) && credits.find((c) => c.file === rel(file))?.ref !== ref; k++) file = join(opts.to, `${stem}-${k}.${ext}`);
|
|
712
|
+
return file;
|
|
713
|
+
}
|
|
714
|
+
function recordCredit(opts, credits, entry) { writeFileSync(creditsPath(opts), JSON.stringify([...credits.filter((c) => c.file !== entry.file), entry], null, 1)); }
|
|
715
|
+
async function getOne(ref, opts) {
|
|
716
|
+
const m = String(ref).match(/^(openverse|commons|aic|met|archive|nasa):(.+)$/);
|
|
717
|
+
if (!m) throw new Error('a ref looks like openverse:<id>, commons:<id>, aic:<id>, met:<id>, archive:<id> or nasa:<id>, as a search prints it.');
|
|
718
|
+
const [, src, id] = m;
|
|
719
|
+
let it;
|
|
720
|
+
if (src === 'openverse') { it = fromOpenverse(await getJson(`https://api.openverse.org/v1/images/${encodeURIComponent(id)}/`, 30000)); await wikimediaSizes([it], opts.width); }
|
|
721
|
+
else if (src === 'commons') {
|
|
722
|
+
// the same ref form serves pictures and videos; the record says which
|
|
723
|
+
const j = await getJson(commonsApi({ pageids: id, ...commonsInfo(opts.width), iiprop: 'url|size|mime|mediatype|extmetadata', prop: 'imageinfo|videoinfo', viprop: 'derivatives' }));
|
|
724
|
+
const pg = j.query?.pages?.[0]; const ii = pg?.imageinfo?.[0];
|
|
725
|
+
it = ii && (ii.mediatype === 'VIDEO' || /^video\//.test(ii.mime || '')) ? fromCommonsVideo(pg, opts) : fromCommons(pg);
|
|
726
|
+
}
|
|
727
|
+
else if (src === 'aic') { const j = await getJson(`https://api.artic.edu/api/v1/artworks/${encodeURIComponent(id)}?fields=${AIC_FIELDS.join(',')}`, 20000, AIC_HEADERS); it = fromAic(j.data, j.config?.iiif_url || 'https://www.artic.edu/iiif/2', opts.width); }
|
|
728
|
+
else if (src === 'archive') it = fromArchive(await getJson(`https://archive.org/metadata/${encodeURIComponent(id)}`, 30000), opts);
|
|
729
|
+
else if (src === 'nasa') {
|
|
730
|
+
const j = await getJson(`https://images-api.nasa.gov/search?${new URLSearchParams({ nasa_id: decodeURIComponent(id), media_type: 'video' })}`);
|
|
731
|
+
const x = j.collection?.items?.[0];
|
|
732
|
+
it = x ? fromNasa(x, await getJson(nasaUrl(x.href), 15000), opts) : null;
|
|
733
|
+
}
|
|
734
|
+
else {
|
|
735
|
+
it = fromMet(await getJson(`https://collectionapi.metmuseum.org/public/collection/v1/objects/${encodeURIComponent(id)}`));
|
|
736
|
+
if (it?.smaller) { const s = await probe(it.smaller); if (s.ok && s.w >= opts.width * 0.9) it.image = it.smaller; }
|
|
737
|
+
}
|
|
738
|
+
if (!it || !it.image) throw new Error(`no usable ${['archive', 'nasa'].includes(src) ? 'video (no rendition a browser plays, or the record is gone)' : 'image (not public domain, or the record has no picture)'}.`);
|
|
739
|
+
if (!opts.licences.includes(it.license)) throw new Error(`its licence is ${licenceLabel(it)}, outside the ones allowed (${opts.licences.join(', ')}). Pass --license all to take it anyway, and say why in your report.`);
|
|
740
|
+
if (it.media === 'video') return getVideo(it, src, id, opts);
|
|
741
|
+
let r;
|
|
742
|
+
try { r = await fetch(it.image, { headers: imageHeaders(it.image), signal: AbortSignal.timeout(90000) }); }
|
|
743
|
+
catch (e) { throw new Error(`the image did not download (${e.name === 'TimeoutError' ? 'timed out' : e.cause?.code || e.message}).`); }
|
|
744
|
+
if (!r.ok) throw new Error(`the image did not load (HTTP ${r.status}).`);
|
|
745
|
+
const type = (r.headers.get('content-type') || '').split(';')[0].trim().toLowerCase();
|
|
746
|
+
const ext = WEB_TYPES[type];
|
|
747
|
+
if (!ext) throw new Error(`the file is ${type || 'of no stated type'}, not a web image.`);
|
|
748
|
+
const buf = Buffer.from(await r.arrayBuffer());
|
|
749
|
+
if (!buf.length) throw new Error('the image was empty.');
|
|
750
|
+
const d = dims(buf);
|
|
751
|
+
mkdirSync(opts.to, { recursive: true });
|
|
752
|
+
const credits = readCredits(opts);
|
|
753
|
+
const stem = slug(args.name || it.title) || `${src}-${slug(id).slice(0, 12)}`;
|
|
754
|
+
const file = targetFile(opts, stem, ext, it.ref, credits);
|
|
755
|
+
writeFileSync(file, buf);
|
|
756
|
+
const entry = {
|
|
757
|
+
file: rel(file), media: 'image', ref: it.ref, title: it.title, creator: it.creator || null, creatorUrl: it.creatorUrl, date: it.date,
|
|
758
|
+
license: licenceLabel(it), licenseCode: it.license, licenseUrl: licenceUrl(it), creditRequired: creditRequired(it),
|
|
759
|
+
page: it.page, via: it.via, image: it.image, width: d.w || it.imageWidth, height: d.h || it.imageHeight, bytes: buf.length,
|
|
760
|
+
credit: credit(it), notes: [...it.notes, ...licenceNotes(it)], saved: new Date().toISOString(),
|
|
761
|
+
};
|
|
762
|
+
recordCredit(opts, credits, entry);
|
|
763
|
+
const alt = it.alt || it.title;
|
|
764
|
+
console.log(`Saved ${entry.file}${entry.width ? ` · ${entry.width} × ${entry.height}` : ''} · ${size(buf.length)}`);
|
|
765
|
+
console.log(` page ${it.page}`);
|
|
766
|
+
console.log(` credit ${entry.credit}`);
|
|
767
|
+
console.log(` markup <img src="${esc(entry.file)}"${entry.width ? ` width="${entry.width}" height="${entry.height}"` : ''} alt="${esc(short(alt, 120))}" loading="lazy" decoding="async">`);
|
|
768
|
+
console.log(' (rewrite the alt to say what the picture shows; drop loading="lazy" on the hero image)');
|
|
769
|
+
for (const n of entry.notes) console.log(` note ${n}`);
|
|
770
|
+
}
|
|
771
|
+
// A clip: check its size against the cap first, stream it to disk (stopping at the cap), then save a still
|
|
772
|
+
// frame from the source as its poster.
|
|
773
|
+
async function getVideo(it, src, id, opts) {
|
|
774
|
+
const cap = opts.maxMb * 1048576;
|
|
775
|
+
const head = await probe(it.image, 20000, true);
|
|
776
|
+
if (!head.ok) throw new Error(`the video did not load (${head.why}).`);
|
|
777
|
+
const side = it.imageWidth && it.imageHeight ? Math.min(it.imageWidth, it.imageHeight) : 0;
|
|
778
|
+
if (head.bytes && head.bytes > cap) throw new Error(`the clip is ${size(head.bytes)}, over the ${opts.maxMb} MB cap. ${side > 480 ? 'Try --res 480 for a smaller rendition, or a shorter clip' : 'Choose a shorter clip'}; pass --max-mb ${Math.ceil(head.bytes / 1048576) + 1} only if the page really needs this one.`);
|
|
779
|
+
mkdirSync(opts.to, { recursive: true });
|
|
780
|
+
const credits = readCredits(opts);
|
|
781
|
+
const stem = slug(args.name || it.title) || `${src}-${slug(decodeURIComponent(id)).slice(0, 12)}`;
|
|
782
|
+
const file = targetFile(opts, stem, VIDEO_TYPES[head.type], it.ref, credits);
|
|
783
|
+
let r;
|
|
784
|
+
try { r = await fetch(it.image, { headers: { 'User-Agent': UA, Accept: 'video/*' }, signal: AbortSignal.timeout(600000) }); }
|
|
785
|
+
catch (e) { throw new Error(`the video did not download (${e.name === 'TimeoutError' ? 'timed out' : e.cause?.code || e.message}).`); }
|
|
786
|
+
if (!r.ok) throw new Error(`the video did not load (HTTP ${r.status}).`);
|
|
787
|
+
const out = createWriteStream(file); const first512 = []; let kept = 0; let n = 0;
|
|
788
|
+
try {
|
|
789
|
+
const reader = r.body.getReader();
|
|
790
|
+
for (;;) {
|
|
791
|
+
const { done, value } = await reader.read(); if (done) break;
|
|
792
|
+
n += value.length;
|
|
793
|
+
if (n > cap) { reader.cancel().catch(() => {}); throw new Error(`the clip passed the ${opts.maxMb} MB cap while downloading; choose a shorter clip or pass a larger --max-mb.`); }
|
|
794
|
+
if (kept < 524288) { first512.push(value); kept += value.length; }
|
|
795
|
+
if (!out.write(value)) await new Promise((res) => out.once('drain', res));
|
|
796
|
+
}
|
|
797
|
+
await new Promise((res, rej) => out.end((e) => (e ? rej(e) : res())));
|
|
798
|
+
} catch (e) { out.destroy(); try { unlinkSync(file); } catch {} throw e; }
|
|
799
|
+
if (!n) { try { unlinkSync(file); } catch {} throw new Error('the video was empty.'); }
|
|
800
|
+
const info = head.type === 'video/mp4' ? mp4Info(Buffer.concat(first512)) : {};
|
|
801
|
+
const w = info.w || it.imageWidth; const h = info.h || it.imageHeight; const duration = info.duration || it.duration;
|
|
802
|
+
let poster = null;
|
|
803
|
+
if (it.poster) {
|
|
804
|
+
try {
|
|
805
|
+
const pr = await fetch(it.poster, { headers: imageHeaders(it.poster), signal: AbortSignal.timeout(60000) });
|
|
806
|
+
const pt = (pr.headers.get('content-type') || '').split(';')[0].trim().toLowerCase();
|
|
807
|
+
if (pr.ok && WEB_TYPES[pt]) { poster = file.replace(/\.[^.]+$/, `-poster.${WEB_TYPES[pt]}`); writeFileSync(poster, Buffer.from(await pr.arrayBuffer())); }
|
|
808
|
+
} catch { poster = null; }
|
|
809
|
+
}
|
|
810
|
+
const entry = {
|
|
811
|
+
file: rel(file), media: 'video', poster: poster ? rel(poster) : null, ref: it.ref, title: it.title, creator: it.creator || null, creatorUrl: it.creatorUrl, date: it.date,
|
|
812
|
+
license: licenceLabel(it), licenseCode: it.license, licenseUrl: licenceUrl(it), creditRequired: creditRequired(it),
|
|
813
|
+
page: it.page, via: it.via, video: it.image, width: w || null, height: h || null, duration: duration ? Math.round(duration * 10) / 10 : null, bytes: n,
|
|
814
|
+
credit: credit(it), notes: [...it.notes, ...licenceNotes(it), ...videoNotes({ ...it, bytes: n, duration }, opts)], saved: new Date().toISOString(),
|
|
815
|
+
};
|
|
816
|
+
recordCredit(opts, credits, entry);
|
|
817
|
+
console.log(`Saved ${entry.file}${w ? ` · ${w} × ${h}` : ''}${duration ? ` · ${clock(duration)}` : ''} · ${size(n)}`);
|
|
818
|
+
console.log(` poster ${entry.poster || 'none from the source: export a still frame yourself, or use a found picture'}`);
|
|
819
|
+
console.log(` page ${it.page}`);
|
|
820
|
+
console.log(` credit ${entry.credit}`);
|
|
821
|
+
console.log(` markup <video autoplay muted loop playsinline preload="metadata"${entry.poster ? ` poster="${esc(entry.poster)}"` : ''}${w ? ` width="${w}" height="${h}"` : ''} aria-hidden="true"><source src="${esc(entry.file)}" type="${head.type}"></video>`);
|
|
822
|
+
console.log(' (a silent loop is decoration: pause it or show the poster under prefers-reduced-motion; references/imagery.md "Video")');
|
|
823
|
+
for (const x of entry.notes) console.log(` note ${x}`);
|
|
824
|
+
}
|
|
825
|
+
async function getAll(refs, opts) {
|
|
826
|
+
let failed = 0;
|
|
827
|
+
for (const ref of refs) {
|
|
828
|
+
try { await getOne(ref, opts); } catch (e) { failed++; console.error(`Could not get ${ref}: ${e.message}`); }
|
|
829
|
+
console.log('');
|
|
830
|
+
}
|
|
831
|
+
if (failed < refs.length) {
|
|
832
|
+
console.log(`Credits are recorded in ${rel(creditsPath(opts))}. Look at each picture and watch each clip before using it.`);
|
|
833
|
+
console.log(`Print the footer credits and the report list with: ${CMD} --credits${opts.toArg}`);
|
|
834
|
+
}
|
|
835
|
+
return failed ? 1 : 0;
|
|
836
|
+
}
|
|
837
|
+
|
|
838
|
+
// ---------- credits ----------
|
|
839
|
+
function printCredits(opts) {
|
|
840
|
+
const list = readCredits(opts);
|
|
841
|
+
const uniq = list.filter((c, i) => list.findIndex((x) => x.ref === c.ref) === i && existsSync(resolve(c.file)));
|
|
842
|
+
if (!uniq.length) { console.log(`No pictures recorded in ${rel(creditsPath(opts))} yet. Download them with --get <ref> first.`); return 1; }
|
|
843
|
+
const html = (c) => {
|
|
844
|
+
const by = c.creator && !/^(unknown|anonymous)\b/i.test(c.creator) ? ` by ${esc(short(c.creator, 60))}` : '';
|
|
845
|
+
const date = /^(aic|met):/.test(c.ref) && c.date ? `, ${esc(short(c.date, 30))}` : '';
|
|
846
|
+
const lic = c.licenseUrl ? `<a href="${esc(c.licenseUrl)}">${esc(c.license)}</a>` : esc(c.license);
|
|
847
|
+
return `<a href="${esc(c.page)}">“${esc(short(c.title, 90))}”</a>${by}${date}, ${lic}, via ${esc(c.via)}`;
|
|
848
|
+
};
|
|
849
|
+
const media = new Set(uniq.map((c) => c.media || 'image'));
|
|
850
|
+
const label = media.size > 1 ? 'Pictures and footage' : media.has('video') ? 'Footage' : 'Pictures';
|
|
851
|
+
console.log('Footer credits (HTML). Full contrast, keep the links, body size (one step smaller only on a page with plenty of');
|
|
852
|
+
console.log('running text). Remove anything the page no longer uses, here and in credits.json.\n');
|
|
853
|
+
console.log(`<p class="credits">${label}: ${uniq.map(html).join('; ')}.</p>`);
|
|
854
|
+
console.log('\nFor your report:\n');
|
|
855
|
+
for (const c of uniq) console.log(`- ${c.file}${c.media === 'video' && c.duration ? ` (video, ${clock(c.duration)})` : ''}: ${c.credit}. ${c.page}${c.creditRequired ? '' : ' (no credit required)'}`);
|
|
856
|
+
return 0;
|
|
857
|
+
}
|
|
858
|
+
|
|
859
|
+
// ---------- run ----------
|
|
860
|
+
if (args.help) { console.log(USAGE); process.exit(0); }
|
|
861
|
+
if (args.credits) process.exit(printCredits(o));
|
|
862
|
+
if (args.get) process.exit(await getAll(String(args.get).split(',').map((s) => s.trim()).filter(Boolean), o));
|
|
863
|
+
const query = words.join(' ').trim();
|
|
864
|
+
if (!query) fail('Give a search query, for example: node assets.mjs "ghent postcard"');
|
|
865
|
+
process.exit(await search(query, o));
|