mlai-code 0.3.2 → 0.3.4
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +1 -1
- package/package.json +1 -1
- package/src/checkpoints.mjs +217 -0
- package/src/cli.mjs +178 -31
- package/src/cloud.mjs +2 -0
- package/src/engine.mjs +1 -1
- package/src/headless.mjs +25 -9
- package/src/home.mjs +3 -2
- package/src/host.mjs +141 -13
- package/src/ide-quick.mjs +147 -0
- package/src/ide.mjs +182 -0
- package/src/netbox.mjs +1 -1
- package/src/parity.mjs +20 -1
- package/src/relay.mjs +1 -1
- package/src/render.mjs +12 -3
- package/src/runtime.mjs +23 -1
- package/src/sandbox.mjs +19 -3
- package/src/server.mjs +46 -7
- package/src/session.mjs +53 -10
- package/src/shared-data.mjs +1 -1
- package/src/tui/acp.mjs +121 -19
- package/src/tui/app.mjs +24 -5
- package/src/tui/chat.mjs +43 -16
- package/src/tui/cmd-config.mjs +3 -1
- package/src/tui/cmd-core.mjs +11 -7
- package/src/tui/cmd-extensibility.mjs +13 -2
- package/src/tui/commands.mjs +2 -1
- package/src/tui/extensions-local.mjs +2 -0
- package/src/tui/mcp-config.mjs +22 -4
- package/src/tui/review-cli.mjs +15 -4
- package/src/tui/snapshots.mjs +57 -16
- package/src/tui/statusline.mjs +2 -1
- package/src/two-words.mjs +14 -8
- package/src/view-sync.mjs +48 -0
- package/src/wakeups.mjs +2 -2
- package/src/winenv.mjs +1 -1
- package/src/workspace-api.mjs +21 -4
- package/vendor/client-sdk/index.cjs +5 -4
- package/vendor/cloud-cli/mlai.mjs +184 -28
- package/worker/architecture_worker.py +33 -4
- package/worker/browser_tool.py +19 -1
- package/worker/build_planner.py +75 -7
- package/worker/chat_fork.py +4 -1
- package/worker/checkpoints.py +71 -8
- package/worker/cloud_pty.py +91 -2
- package/worker/command_safety.py +80 -3
- package/worker/compaction_brief.py +186 -0
- package/worker/connected_turn.py +30 -0
- package/worker/context_manager.py +6 -0
- package/worker/conversation.py +139 -12
- package/worker/deck_render.js +11 -0
- package/worker/deck_tools.py +825 -0
- package/worker/git_sync.py +42 -17
- package/worker/goals.py +4 -0
- package/worker/image_generation.py +102 -23
- package/worker/kimi_bridge.py +580 -52
- package/worker/kimi_gates.py +74 -6
- package/worker/kimi_hook.py +213 -19
- package/worker/media_tools.py +182 -0
- package/worker/memory.py +2 -2
- package/worker/mlai_hooks.py +314 -5
- package/worker/mlai_mcp.py +49 -11
- package/worker/mlai_review/README.md +35 -0
- package/worker/mlai_review/__init__.py +1 -0
- package/worker/mlai_review/__main__.py +261 -0
- package/worker/mlai_review/_compat.py +29 -0
- package/worker/mlai_review/analysis/__init__.py +3 -0
- package/worker/mlai_review/analysis/detect.py +50 -0
- package/worker/mlai_review/analysis/main.py +128 -0
- package/worker/mlai_review/analysis/messages.py +8 -0
- package/worker/mlai_review/analysis/tools.py +194 -0
- package/worker/mlai_review/application/__init__.py +1 -0
- package/worker/mlai_review/application/asking.py +53 -0
- package/worker/mlai_review/application/challengers.py +142 -0
- package/worker/mlai_review/application/context_builder.py +194 -0
- package/worker/mlai_review/application/errors.py +17 -0
- package/worker/mlai_review/application/feedback.py +52 -0
- package/worker/mlai_review/application/matching.py +46 -0
- package/worker/mlai_review/application/messages.py +41 -0
- package/worker/mlai_review/application/ports.py +222 -0
- package/worker/mlai_review/application/prompts.py +342 -0
- package/worker/mlai_review/application/review_service.py +200 -0
- package/worker/mlai_review/application/reviewers.py +438 -0
- package/worker/mlai_review/application/streaming.py +91 -0
- package/worker/mlai_review/application/tools.py +138 -0
- package/worker/mlai_review/application/usage.py +145 -0
- package/worker/mlai_review/application/write_check.py +46 -0
- package/worker/mlai_review/contracts/__init__.py +5 -0
- package/worker/mlai_review/contracts/analysis.py +57 -0
- package/worker/mlai_review/contracts/review.py +42 -0
- package/worker/mlai_review/domain/__init__.py +4 -0
- package/worker/mlai_review/domain/changes.py +77 -0
- package/worker/mlai_review/domain/context.py +147 -0
- package/worker/mlai_review/domain/diagnostics.py +239 -0
- package/worker/mlai_review/domain/feedback.py +216 -0
- package/worker/mlai_review/domain/findings.py +505 -0
- package/worker/mlai_review/domain/lifecycle.py +85 -0
- package/worker/mlai_review/domain/partition.py +42 -0
- package/worker/mlai_review/domain/relations.py +166 -0
- package/worker/mlai_review/domain/relevance.py +202 -0
- package/worker/mlai_review/domain/report.py +99 -0
- package/worker/mlai_review/domain/structure.py +56 -0
- package/worker/mlai_review/domain/writes.py +86 -0
- package/worker/mlai_review/eval.py +221 -0
- package/worker/mlai_review/eval_cases/clean-rename.json +11 -0
- package/worker/mlai_review/eval_cases/missing-ownership-check.json +22 -0
- package/worker/mlai_review/eval_cases/off-by-one.json +22 -0
- package/worker/mlai_review/eval_cases/sql-injection.json +20 -0
- package/worker/mlai_review/feedback_report.py +60 -0
- package/worker/mlai_review/infrastructure/__init__.py +1 -0
- package/worker/mlai_review/infrastructure/analysis/__init__.py +1 -0
- package/worker/mlai_review/infrastructure/analysis/none.py +12 -0
- package/worker/mlai_review/infrastructure/analysis/translate.py +17 -0
- package/worker/mlai_review/infrastructure/github/__init__.py +0 -0
- package/worker/mlai_review/infrastructure/github/client.py +445 -0
- package/worker/mlai_review/infrastructure/github/feedback.py +212 -0
- package/worker/mlai_review/infrastructure/github/http.py +31 -0
- package/worker/mlai_review/infrastructure/github/rendering.py +398 -0
- package/worker/mlai_review/infrastructure/github/token.py +20 -0
- package/worker/mlai_review/infrastructure/local/__init__.py +1 -0
- package/worker/mlai_review/infrastructure/local/analyzer.py +48 -0
- package/worker/mlai_review/infrastructure/local/events.py +100 -0
- package/worker/mlai_review/infrastructure/local/git_reader.py +233 -0
- package/worker/mlai_review/infrastructure/local/publisher.py +138 -0
- package/worker/mlai_review/infrastructure/messages.py +54 -0
- package/worker/mlai_review/infrastructure/models/__init__.py +0 -0
- package/worker/mlai_review/infrastructure/models/openai_compatible.py +269 -0
- package/worker/mlai_review/infrastructure/structure/__init__.py +1 -0
- package/worker/mlai_review/infrastructure/structure/heuristic.py +186 -0
- package/worker/mlai_review/infrastructure/structure/tree_sitter.py +186 -0
- package/worker/mlai_review/infrastructure/workspace/__init__.py +1 -0
- package/worker/mlai_review/infrastructure/workspace/local.py +105 -0
- package/worker/mlai_review/infrastructure/workspace/memory.py +77 -0
- package/worker/mlai_review/infrastructure/workspace/paths.py +17 -0
- package/worker/mlai_review/pipeline.py +89 -0
- package/worker/mlai_review/runtime.py +194 -0
- package/worker/mlai_review/settings.py +139 -0
- package/worker/model_stream.py +21 -7
- package/worker/offline_guard.py +1 -1
- package/worker/preview.py +62 -4
- package/worker/product_facts.py +20 -0
- package/worker/project_context.py +33 -7
- package/worker/reply_language.py +92 -10
- package/worker/repo_policy.py +5 -0
- package/worker/roles.json +199 -37
- package/worker/roles.py +271 -9
- package/worker/runner.py +366 -53
- package/worker/runtime_control.py +64 -3
- package/worker/secret_scan.py +3 -2
- package/worker/server_memory.py +122 -0
- package/worker/site_check.py +279 -0
- package/worker/skill_library.py +15 -3
- package/worker/skill_pack.py +170 -16
- package/worker/subagents.py +6 -2
- package/worker/tool_catalog.py +4 -3
- package/worker/verification.py +156 -19
- package/worker/wakeups.py +56 -4
|
@@ -0,0 +1,825 @@
|
|
|
1
|
+
"""Deck quality tools: the slide check (render every slide, find crowding, overflow, overlaps, tiny type, contrast and broken
|
|
2
|
+
images; fix what it can; export a PDF), PowerPoint template and spreadsheet import, free images with credits (Openverse,
|
|
3
|
+
Wikimedia Commons), icons and brand logos (Iconify), and image preparation (cut-out, smart crop, darken, mono, duotone).
|
|
4
|
+
|
|
5
|
+
The check renders with the app's own slide renderer (worker/deck_render.js, built from scripts/deck-render-entry.tsx) in
|
|
6
|
+
headless Chromium. The page is served from a fake origin with every request blocked except Google Fonts; images are
|
|
7
|
+
inlined, so nothing in a deck can reach the network. Image and icon search need no keys and call public APIs directly.
|
|
8
|
+
"""
|
|
9
|
+
import base64, csv, io, ipaddress, json, os, re, socket, urllib.parse, urllib.request, urllib.error, zipfile
|
|
10
|
+
import xml.etree.ElementTree as ET
|
|
11
|
+
from pathlib import Path
|
|
12
|
+
|
|
13
|
+
HERE = Path(__file__).resolve().parent
|
|
14
|
+
ORIGIN = 'http://deck.local'
|
|
15
|
+
FONT_HOSTS = ('fonts.googleapis.com', 'fonts.gstatic.com')
|
|
16
|
+
UA = 'ml-ai-code/1 (+https://code.ml.ai)'
|
|
17
|
+
MAX_IMAGE = 12_000_000
|
|
18
|
+
MAX_DECK_IMAGES = 80_000_000
|
|
19
|
+
MAX_IMPORT = 120_000_000
|
|
20
|
+
CAPS = {'find_image': 30, 'find_icon': 60, 'deck_check': 40, 'image_prep': 40, 'deck_import': 10}
|
|
21
|
+
OPENVERSE = 'https://api.openverse.org/v1/images/'
|
|
22
|
+
COMMONS = 'https://commons.wikimedia.org/w/api.php'
|
|
23
|
+
ICONIFY = 'https://api.iconify.design'
|
|
24
|
+
ICON_SETS = ('lucide', 'tabler', 'mdi', 'simple-icons', 'logos', 'ph')
|
|
25
|
+
OPEN_LICENSES = re.compile(r'^(cc0|pdm|public domain|pd|cc[- ]by(?:[- ]sa)?(?:[- ][\d.]+)?)', re.I)
|
|
26
|
+
CAPACITY = {'bullets': 6, 'agenda': 8, 'split': 5, 'features': 6, 'steps': 5, 'timeline': 6, 'team': 6, 'logos': 12, 'gallery': 4, 'compare': 3, 'columns': 4, 'stats': 4, 'table': 8}
|
|
27
|
+
|
|
28
|
+
DECK_CHECK_TOOL = {'name': 'deck_check', 'description': 'Render every slide of a deck (decks/*.deck.json) and check it the way a designer would: text shrunk to fit or cut off, overlapping text, text too small to read, text over photos with poor contrast, broken images, over-full slides, topic titles, chart and diagram data problems, missing sources and notes. Returns the issues and shows you a contact sheet of the slides. fix=true also splits over-full slides, crops photos around their subject and darkens photos under hard-to-read text, then re-checks. pdf=true also writes exports/<deck>.pdf. slides=[n] zooms in on those slides.',
|
|
29
|
+
'inputSchema': {'type': 'object', 'properties': {'path': {'type': 'string', 'description': 'Deck file, e.g. decks/pitch.deck.json'}, 'fix': {'type': 'boolean'}, 'pdf': {'type': 'boolean'},
|
|
30
|
+
'slides': {'type': 'array', 'items': {'type': 'integer', 'minimum': 1}, 'maxItems': 8}}, 'required': ['path'], 'additionalProperties': False}}
|
|
31
|
+
DECK_IMPORT_TOOL = {'name': 'deck_import', 'description': "Import a file for a deck. A PowerPoint (.pptx) template or deck: its theme colours, fonts, logo and master background become a deck theme; mode=deck also converts its slides (text, bullets, tables, charts, pictures, speaker notes) into an editable decks/<name>.deck.json. A spreadsheet (.xlsx or .csv): returns chart-ready labels and series (first column = labels, numeric columns = series) for a chart slide.",
|
|
32
|
+
'inputSchema': {'type': 'object', 'properties': {'path': {'type': 'string', 'description': 'Workspace file, e.g. uploads/brand-template.pptx or data/revenue.xlsx'},
|
|
33
|
+
'mode': {'type': 'string', 'enum': ['theme', 'deck'], 'description': 'pptx only: theme (default) or deck'}, 'sheet': {'type': 'string', 'description': 'xlsx only: sheet name (default: the first)'},
|
|
34
|
+
'range': {'type': 'string', 'description': 'xlsx/csv only: A1-style range such as A1:D13'}}, 'required': ['path'], 'additionalProperties': False}}
|
|
35
|
+
FIND_IMAGE_TOOL = {'name': 'find_image', 'description': 'Find a freely licensed photo or illustration (CC0, public domain, CC BY, CC BY-SA) on Openverse or Wikimedia Commons and save it under public/stock/ with its credit (free, no credits). Best for real places, landmarks, historical images, people in the news, products, science and nature; stock_photo is better for generic business photos. Returns the saved path and a credit line to put in the slide\'s credit field, plus other matches (call again with pick).',
|
|
36
|
+
'inputSchema': {'type': 'object', 'properties': {'query': {'type': 'string', 'maxLength': 100}, 'filename': {'type': 'string', 'description': 'New two-word kebab filename'},
|
|
37
|
+
'source': {'type': 'string', 'enum': ['auto', 'openverse', 'wikimedia']}, 'pick': {'type': 'string', 'description': 'id of an earlier match'}}, 'required': ['query', 'filename'], 'additionalProperties': False}}
|
|
38
|
+
FIND_ICON_TOOL = {'name': 'find_icon', 'description': 'Find an icon or brand logo among about 200,000 open icons (Lucide, Tabler, Material Design, Phosphor, Simple Icons and SVG Logos for company and product logos) and save it as an SVG under public/icons/ in the colour you ask for. Use the saved path as an item icon or diagram node icon, or a logos-slide image. name="simple-icons:stripe" fetches an exact icon.',
|
|
39
|
+
'inputSchema': {'type': 'object', 'properties': {'query': {'type': 'string', 'maxLength': 60}, 'name': {'type': 'string', 'maxLength': 80}, 'color': {'type': 'string', 'description': 'Hex colour, e.g. #8b5cf6 (brand logos keep their own colours when omitted)'}},
|
|
40
|
+
'additionalProperties': False}}
|
|
41
|
+
IMAGE_PREP_TOOL = {'name': 'image_prep', 'description': 'Prepare an image for a slide and save the result as a new file: cutout (remove a flat, plain background to a transparent PNG; busy photo backgrounds are reported, not guessed), crop (to an aspect ratio around the subject), focus (just report where the subject is, for imageFocus), darken, mono or duotone (in the given colours).',
|
|
42
|
+
'inputSchema': {'type': 'object', 'properties': {'path': {'type': 'string'}, 'op': {'type': 'string', 'enum': ['cutout', 'crop', 'focus', 'darken', 'mono', 'duotone']},
|
|
43
|
+
'aspect': {'type': 'string', 'description': 'crop only: 16:9, 4:3, 1:1, 3:4 or 9:16'}, 'dark': {'type': 'string'}, 'bright': {'type': 'string'}}, 'required': ['path', 'op'], 'additionalProperties': False}}
|
|
44
|
+
TOOLS = [DECK_CHECK_TOOL, DECK_IMPORT_TOOL, FIND_IMAGE_TOOL, FIND_ICON_TOOL, IMAGE_PREP_TOOL]
|
|
45
|
+
NAMES = tuple(t['name'] for t in TOOLS)
|
|
46
|
+
READ_ONLY = ('deck_check',) # without fix or pdf it only reads the deck
|
|
47
|
+
|
|
48
|
+
|
|
49
|
+
# ---------- shared helpers ----------
|
|
50
|
+
|
|
51
|
+
def _stem(name):
|
|
52
|
+
stem = re.sub(r'[^a-z0-9-]+', '-', str(name or '').lower().rsplit('.', 1)[0]).strip('-')[:48]
|
|
53
|
+
if not stem: raise ValueError('filename is required, e.g. golden-gate')
|
|
54
|
+
return stem
|
|
55
|
+
|
|
56
|
+
def _inside(repo, rel):
|
|
57
|
+
"""A workspace path that stays inside the repository (no absolute paths, no .. escapes, no symlinked parents)."""
|
|
58
|
+
raw = str(rel or '').strip().lstrip('/')
|
|
59
|
+
if not raw: raise ValueError('path is required')
|
|
60
|
+
repo = Path(repo).resolve(); target = (repo / raw).resolve()
|
|
61
|
+
if target != repo and repo not in target.parents: raise ValueError('Path must be inside the project')
|
|
62
|
+
return target
|
|
63
|
+
|
|
64
|
+
def _public(repo, name):
|
|
65
|
+
public = Path(repo) / 'public'; d = public / name
|
|
66
|
+
for p in (public, d):
|
|
67
|
+
if p.is_symlink(): raise ValueError('Asset directory cannot be a symlink')
|
|
68
|
+
d.mkdir(parents=True, exist_ok=True); return d
|
|
69
|
+
|
|
70
|
+
def _count(root, kind):
|
|
71
|
+
path = Path(root) / 'deck-tools-usage.json'
|
|
72
|
+
try: used = json.loads(path.read_text())
|
|
73
|
+
except (FileNotFoundError, ValueError): used = {}
|
|
74
|
+
if used.get(kind, 0) >= CAPS[kind]: raise ValueError('%s limit reached (%d per task). Reuse what you have.' % (kind, CAPS[kind]))
|
|
75
|
+
used[kind] = used.get(kind, 0) + 1
|
|
76
|
+
tmp = path.with_suffix('.tmp'); tmp.write_text(json.dumps(used)); tmp.replace(path)
|
|
77
|
+
|
|
78
|
+
def _public_host(url):
|
|
79
|
+
"""https only, and the host must resolve to public addresses (no loopback, private or link-local targets)."""
|
|
80
|
+
p = urllib.parse.urlparse(str(url or ''))
|
|
81
|
+
if p.scheme != 'https' or not p.hostname or p.username or p.password: return False
|
|
82
|
+
try: infos = socket.getaddrinfo(p.hostname, 443, proto=socket.IPPROTO_TCP)
|
|
83
|
+
except OSError: return False
|
|
84
|
+
for info in infos:
|
|
85
|
+
ip = ipaddress.ip_address(info[4][0])
|
|
86
|
+
if ip.is_private or ip.is_loopback or ip.is_link_local or ip.is_multicast or ip.is_reserved or ip.is_unspecified: return False
|
|
87
|
+
return True
|
|
88
|
+
|
|
89
|
+
class _NoRedirect(urllib.request.HTTPRedirectHandler):
|
|
90
|
+
def redirect_request(self, req, fp, code, msg, headers, newurl):
|
|
91
|
+
if not _public_host(newurl): raise ValueError('Redirect to a non-public address was blocked')
|
|
92
|
+
return super().redirect_request(req, fp, code, msg, headers, newurl)
|
|
93
|
+
|
|
94
|
+
def _get(url, limit=3_000_000, timeout=30, accept=None):
|
|
95
|
+
if not _public_host(url): raise ValueError('Only public https addresses can be fetched')
|
|
96
|
+
req = urllib.request.Request(url, headers={'User-Agent': UA, **({'Accept': accept} if accept else {})})
|
|
97
|
+
with urllib.request.build_opener(urllib.request.ProxyHandler({}), _NoRedirect()).open(req, timeout=timeout) as r:
|
|
98
|
+
data = r.read(limit + 1)
|
|
99
|
+
if len(data) > limit: raise ValueError('Download too large')
|
|
100
|
+
return data
|
|
101
|
+
|
|
102
|
+
def _offline():
|
|
103
|
+
import offline_guard
|
|
104
|
+
offline_guard.check('media')
|
|
105
|
+
|
|
106
|
+
def image_kind(data):
|
|
107
|
+
if data[:3] == b'\xff\xd8\xff': return 'jpeg'
|
|
108
|
+
if data[:8] == b'\x89PNG\r\n\x1a\n': return 'png'
|
|
109
|
+
if data[:4] == b'RIFF' and data[8:12] == b'WEBP': return 'webp'
|
|
110
|
+
if data[:6] in (b'GIF87a', b'GIF89a'): return 'gif'
|
|
111
|
+
head = data[:2000].lstrip().lower()
|
|
112
|
+
if head.startswith(b'<svg') or (head.startswith(b'<?xml') and b'<svg' in head): return 'svg'
|
|
113
|
+
return None
|
|
114
|
+
|
|
115
|
+
def data_url(data):
|
|
116
|
+
kind = image_kind(data)
|
|
117
|
+
if not kind: return None
|
|
118
|
+
return 'data:image/%s;base64,%s' % ('svg+xml' if kind == 'svg' else kind, base64.b64encode(data).decode())
|
|
119
|
+
|
|
120
|
+
def resolve_image(deck_path, image):
|
|
121
|
+
"""Same rules as lib/deck.ts resolveDeckPath: workspace-root paths stay as they are, others are relative to the deck."""
|
|
122
|
+
if re.match(r'^(https?:|data:)', image): return image
|
|
123
|
+
clean = image.lstrip('/')
|
|
124
|
+
if not clean.startswith('./') and not clean.startswith('../'):
|
|
125
|
+
if re.match(r'^(public|decks|assets|src|docs|reports|images)/', clean): return clean
|
|
126
|
+
return (deck_path.rsplit('/', 1)[0] + '/' if '/' in deck_path else '') + clean
|
|
127
|
+
parts = deck_path.split('/')[:-1]
|
|
128
|
+
for seg in clean.split('/'):
|
|
129
|
+
if seg in ('.', ''): continue
|
|
130
|
+
if seg == '..':
|
|
131
|
+
if parts: parts.pop()
|
|
132
|
+
else: parts.append(seg)
|
|
133
|
+
return '/'.join(parts)
|
|
134
|
+
|
|
135
|
+
def deck_image_paths(deck):
|
|
136
|
+
t = deck.get('theme') or {}; out = [t.get('logo'), t.get('texture')]
|
|
137
|
+
for s in deck.get('slides') or []:
|
|
138
|
+
out += [s.get('image'), s.get('background')]
|
|
139
|
+
for it in s.get('items') or []:
|
|
140
|
+
out.append(it.get('image'))
|
|
141
|
+
if it.get('icon') and re.search(r'[/.]', it['icon']): out.append(it['icon'])
|
|
142
|
+
for n in ((s.get('diagram') or {}).get('nodes') or []):
|
|
143
|
+
if n.get('icon') and re.search(r'[/.]', n['icon']): out.append(n['icon'])
|
|
144
|
+
return [p for p in dict.fromkeys(out) if isinstance(p, str) and p]
|
|
145
|
+
|
|
146
|
+
|
|
147
|
+
# ---------- the slide check ----------
|
|
148
|
+
|
|
149
|
+
def _bundle():
|
|
150
|
+
path = HERE / 'deck_render.js'
|
|
151
|
+
if not path.exists(): raise ValueError('The slide renderer is missing from this workspace (deck_render.js).')
|
|
152
|
+
return path.read_bytes()
|
|
153
|
+
|
|
154
|
+
PAGE = ('<!doctype html><html><head><meta charset="utf-8"><style>html,body{margin:0;background:#fff}'
|
|
155
|
+
'.deck-notext .dk-slide *{color:transparent!important;-webkit-text-fill-color:transparent!important;text-shadow:none!important}'
|
|
156
|
+
'.deck-notext .dk-slide text{fill:transparent!important}</style></head><body><div id="root"></div><script src="/render.js"></script></body></html>')
|
|
157
|
+
|
|
158
|
+
class Browser:
|
|
159
|
+
"""Headless Chromium with the slide renderer loaded. Only the fake origin and Google Fonts are reachable."""
|
|
160
|
+
def __init__(self, root):
|
|
161
|
+
self.root = Path(root)
|
|
162
|
+
def __enter__(self):
|
|
163
|
+
import browser_local
|
|
164
|
+
local = browser_local.laptop()
|
|
165
|
+
if not (local and os.getenv('PLAYWRIGHT_BROWSERS_PATH')): os.environ['PLAYWRIGHT_BROWSERS_PATH'] = str(self.root / 'pw-browsers')
|
|
166
|
+
from playwright.sync_api import sync_playwright
|
|
167
|
+
self._pw = sync_playwright().start()
|
|
168
|
+
args = ['--no-sandbox']
|
|
169
|
+
if local: args.append('--host-resolver-rules=MAP * ~NOTFOUND , EXCLUDE localhost , EXCLUDE 127.0.0.1 , EXCLUDE fonts.googleapis.com , EXCLUDE fonts.gstatic.com')
|
|
170
|
+
exe = os.getenv('MINI_DECK_CHROMIUM') or None
|
|
171
|
+
try: self.browser = self._pw.chromium.launch(headless=True, args=args, executable_path=exe)
|
|
172
|
+
except Exception: self._pw.stop(); raise
|
|
173
|
+
ctx = self.browser.new_context(viewport={'width': 1280, 'height': 720}, device_scale_factor=1, service_workers='block', accept_downloads=False)
|
|
174
|
+
bundle = _bundle()
|
|
175
|
+
def route(r):
|
|
176
|
+
u = urllib.parse.urlparse(r.request.url)
|
|
177
|
+
if r.request.url.startswith(ORIGIN + '/render.js'): return r.fulfill(status=200, content_type='text/javascript', body=bundle)
|
|
178
|
+
if r.request.url.startswith(ORIGIN): return r.fulfill(status=200, content_type='text/html', body=PAGE)
|
|
179
|
+
if u.scheme == 'https' and u.hostname in FONT_HOSTS: return r.continue_()
|
|
180
|
+
if u.scheme == 'data': return r.continue_()
|
|
181
|
+
return r.abort()
|
|
182
|
+
ctx.route('**/*', route)
|
|
183
|
+
self.page = ctx.new_page(); self.page.set_default_timeout(30000)
|
|
184
|
+
self.errors = []; self.page.on('pageerror', lambda e: self.errors.append(str(e)[:300]))
|
|
185
|
+
self.page.goto(ORIGIN + '/', wait_until='load')
|
|
186
|
+
self.page.wait_for_function('() => typeof window.__deckRender === "function"', timeout=15000)
|
|
187
|
+
return self
|
|
188
|
+
def __exit__(self, *exc):
|
|
189
|
+
try: self.browser.close()
|
|
190
|
+
finally: self._pw.stop()
|
|
191
|
+
|
|
192
|
+
def browser_ready(root):
|
|
193
|
+
return (Path(root) / 'browser-ready').exists() or bool(os.getenv('MINI_DECK_CHROMIUM'))
|
|
194
|
+
|
|
195
|
+
def _load_deck(repo, rel):
|
|
196
|
+
path = _inside(repo, rel)
|
|
197
|
+
if not path.name.endswith('.json') or not path.is_file(): raise ValueError('Give a deck file such as decks/pitch.deck.json')
|
|
198
|
+
text = path.read_text()
|
|
199
|
+
try: deck = json.loads(text)
|
|
200
|
+
except ValueError as exc: raise ValueError('The deck is not valid JSON: %s' % exc) from None
|
|
201
|
+
if not isinstance(deck, dict) or not isinstance(deck.get('slides'), list): raise ValueError('Not a deck: it needs a slides list')
|
|
202
|
+
return path, text, deck
|
|
203
|
+
|
|
204
|
+
def _images(repo, rel, deck):
|
|
205
|
+
out, missing, total = {}, [], 0
|
|
206
|
+
for p in deck_image_paths(deck):
|
|
207
|
+
r = resolve_image(rel, p)
|
|
208
|
+
if r.startswith(('http:', 'https:', 'data:')): continue
|
|
209
|
+
try: f = _inside(repo, r)
|
|
210
|
+
except ValueError: missing.append(p); continue
|
|
211
|
+
if not f.is_file(): missing.append(p); continue
|
|
212
|
+
data = f.read_bytes()
|
|
213
|
+
if len(data) > MAX_IMAGE or total + len(data) > MAX_DECK_IMAGES: continue
|
|
214
|
+
url = data_url(data)
|
|
215
|
+
if url: out[p] = url; total += len(data)
|
|
216
|
+
else: missing.append(p)
|
|
217
|
+
return out, missing
|
|
218
|
+
|
|
219
|
+
def _quick_lint(deck):
|
|
220
|
+
"""Checks without a browser (the browser lint is fuller)."""
|
|
221
|
+
out = []
|
|
222
|
+
for i, s in enumerate(deck.get('slides') or []):
|
|
223
|
+
lay = s.get('layout'); cap = CAPACITY.get(lay)
|
|
224
|
+
n = len(s.get('bullets') or []) if lay in ('bullets', 'agenda', 'split') else len(s.get('columns') or []) if lay == 'columns' else len(s.get('stats') or []) if lay == 'stats' else len((s.get('table') or {}).get('rows') or []) if lay == 'table' else len(s.get('items') or [])
|
|
225
|
+
if cap and n > cap: out.append({'slide': i + 1, 'severity': 'error', 'code': 'over-capacity', 'message': '%d entries on a %s slide (fits %d); split it.' % (n, lay, cap)})
|
|
226
|
+
if len(str(s.get('title') or '')) > 90: out.append({'slide': i + 1, 'severity': 'warn', 'code': 'long-title', 'message': 'Title over 90 characters.'})
|
|
227
|
+
if not s.get('notes') and lay not in ('cover', 'section', 'closing'): out.append({'slide': i + 1, 'severity': 'info', 'code': 'no-notes', 'message': 'No speaker notes.'})
|
|
228
|
+
return out
|
|
229
|
+
|
|
230
|
+
def _render_issues(rep, contrast):
|
|
231
|
+
out = []
|
|
232
|
+
for m in rep:
|
|
233
|
+
n = m['index'] + 1
|
|
234
|
+
if m['over']: out.append({'slide': n, 'severity': 'error', 'code': 'text-cut', 'message': 'Text does not fit even at the smallest size; cut it or split the slide.'})
|
|
235
|
+
elif m['fit'] < .6: out.append({'slide': n, 'severity': 'error', 'code': 'text-shrunk', 'message': 'Text is shrunk to %d%% to fit; split the slide or cut words.' % round(m['fit'] * 100)})
|
|
236
|
+
elif m['fit'] < .8: out.append({'slide': n, 'severity': 'warn', 'code': 'text-shrunk', 'message': 'Text is shrunk to %d%% to fit; it reads as crowded.' % round(m['fit'] * 100)})
|
|
237
|
+
for t in m['overflow'][:3]: out.append({'slide': n, 'severity': 'error', 'code': 'overflow', 'message': 'Text runs outside its box or the slide: "%s".' % t})
|
|
238
|
+
for t in m['overlaps'][:3]: out.append({'slide': n, 'severity': 'error', 'code': 'overlap', 'message': 'Overlapping text: %s.' % t})
|
|
239
|
+
for t in m['tiny'][:3]: out.append({'slide': n, 'severity': 'warn', 'code': 'tiny-text', 'message': 'Text too small to read when presented: %s.' % t})
|
|
240
|
+
if m['broken']: out.append({'slide': n, 'severity': 'error', 'code': 'broken-image', 'message': '%d image(s) failed to load.' % m['broken']})
|
|
241
|
+
for c in contrast.get(m['index'], [])[:3]:
|
|
242
|
+
out.append({'slide': n, 'severity': 'error' if c['ratio'] < c['need'] - 1 else 'warn', 'code': 'low-contrast',
|
|
243
|
+
'message': '"%s" has contrast %.1f:1 against what is behind it (needs %.1f:1%s).' % (c['text'], c['ratio'], c['need'], ', busy photo' if c['busy'] else '')})
|
|
244
|
+
return out
|
|
245
|
+
|
|
246
|
+
def _measure(b, text, images):
|
|
247
|
+
page = b.page
|
|
248
|
+
page.evaluate('([t, i]) => window.__deckRender(t, i)', [text, images])
|
|
249
|
+
rep = page.evaluate('() => window.__deckMeasure().map(m => ({...m, texts: undefined}))')
|
|
250
|
+
contrast = {}
|
|
251
|
+
for i, el in enumerate(page.query_selector_all('.dk-slide')):
|
|
252
|
+
# Shoot the slide with its text hidden, then compare each text's colour with the pixels behind it.
|
|
253
|
+
page.evaluate('() => document.body.classList.add("deck-notext")')
|
|
254
|
+
shot = el.screenshot(type='jpeg', quality=80)
|
|
255
|
+
page.evaluate('() => document.body.classList.remove("deck-notext")')
|
|
256
|
+
found = page.evaluate('([i, s]) => window.__deckContrast(i, s)', [i, 'data:image/jpeg;base64,' + base64.b64encode(shot).decode()])
|
|
257
|
+
if found: contrast[i] = found
|
|
258
|
+
return rep, contrast
|
|
259
|
+
|
|
260
|
+
def _sheet(b, text, images, out, only=None):
|
|
261
|
+
cols = 2 if only and len(only) <= 8 else 3
|
|
262
|
+
b.page.evaluate('([t, i, o]) => window.__deckRender(t, i, o)', [text, images, {'sheet': True, 'cols': cols, **({'only': only} if only else {})}])
|
|
263
|
+
el = b.page.query_selector('#deck')
|
|
264
|
+
el.screenshot(path=str(out), type='jpeg', quality=72)
|
|
265
|
+
return out
|
|
266
|
+
|
|
267
|
+
def _remap(old, new, only):
|
|
268
|
+
"""Slide numbers asked for before a fix, in the fixed deck: a split slide becomes all of its parts."""
|
|
269
|
+
where, j = {}, 0
|
|
270
|
+
for i, s in enumerate(old):
|
|
271
|
+
title = str(s.get('title') or ''); parts = [j]; j += 1
|
|
272
|
+
part = re.compile(re.escape(title) + r' \(\d+/\d+\)$')
|
|
273
|
+
if j - 1 < len(new) and part.match(str(new[j - 1].get('title') or '')):
|
|
274
|
+
while j < len(new) and part.match(str(new[j].get('title') or '')): parts.append(j); j += 1
|
|
275
|
+
where[i + 1] = [k + 1 for k in parts if k < len(new)]
|
|
276
|
+
return list(dict.fromkeys(n for o in only for n in where.get(o, [o])))[:8] or None
|
|
277
|
+
|
|
278
|
+
def deck_check(args, root, repo, ensure_browser=None):
|
|
279
|
+
ensure_browser = ensure_browser or (lambda: browser_ready(root))
|
|
280
|
+
rel = str(args.get('path') or '').strip().lstrip('/')
|
|
281
|
+
path, text, deck = _load_deck(repo, rel)
|
|
282
|
+
fix, pdf = bool(args.get('fix')), bool(args.get('pdf'))
|
|
283
|
+
only = [int(n) for n in (args.get('slides') or []) if isinstance(n, int) and 1 <= n <= len(deck['slides'])][:8] or None
|
|
284
|
+
_count(root, 'deck_check')
|
|
285
|
+
images, missing = _images(repo, rel, deck)
|
|
286
|
+
result = {'path': rel, 'slides': len(deck['slides'])}
|
|
287
|
+
if not ensure_browser():
|
|
288
|
+
issues = _quick_lint(deck) + [{'slide': 0, 'severity': 'error', 'code': 'missing-image', 'message': 'Image not found: %s.' % p} for p in missing]
|
|
289
|
+
result.update(issues=issues, note='The test browser is not available here, so only the file was checked (no render). Re-run when the browser is set up.')
|
|
290
|
+
return result, None
|
|
291
|
+
shots = Path(root) / 'screenshots'; shots.mkdir(exist_ok=True); ident = os.urandom(6).hex(); changes = []
|
|
292
|
+
with Browser(root) as b:
|
|
293
|
+
rep, contrast = _measure(b, text, images)
|
|
294
|
+
lint = b.page.evaluate('([t, p]) => window.__deckLint(t, p)', [text, list(images.keys())])
|
|
295
|
+
if fix:
|
|
296
|
+
focus = b.page.evaluate('(i) => window.__deckFocus(i)', images)
|
|
297
|
+
fixed = b.page.evaluate('([t, r, f]) => window.__deckFix(t, r, f)', [text, {'slides': rep, 'contrast': {str(k): v for k, v in contrast.items()}}, focus])
|
|
298
|
+
changes = fixed['changes']
|
|
299
|
+
if changes and fixed['text'] != text:
|
|
300
|
+
old = deck['slides']
|
|
301
|
+
tmp = path.with_suffix('.tmp'); tmp.write_text(fixed['text']); tmp.replace(path); text = fixed['text']; deck = json.loads(text)
|
|
302
|
+
if only: only = _remap(old, deck['slides'], only)
|
|
303
|
+
rep, contrast = _measure(b, text, images)
|
|
304
|
+
lint = b.page.evaluate('([t, p]) => window.__deckLint(t, p)', [text, list(images.keys())])
|
|
305
|
+
issues = sorted(_render_issues(rep, contrast) + list(lint), key=lambda x: (x['slide'], {'error': 0, 'warn': 1, 'info': 2}[x['severity']]))
|
|
306
|
+
shown = only
|
|
307
|
+
if not shown and len(deck['slides']) > 24:
|
|
308
|
+
# A readable sheet holds about 24 slides: slides with problems first, then the rest in order.
|
|
309
|
+
bad = sorted({x['slide'] for x in issues if x['severity'] in ('error', 'warn') and x['slide']})
|
|
310
|
+
shown = sorted((bad + [n for n in range(1, len(deck['slides']) + 1) if n not in bad])[:24])
|
|
311
|
+
sheet = _sheet(b, text, images, shots / ('deck-%s.jpg' % ident), shown)
|
|
312
|
+
if pdf:
|
|
313
|
+
b.page.evaluate('([t, i]) => window.__deckRender(t, i)', [text, images])
|
|
314
|
+
out = _inside(repo, 'exports/' + path.name.replace('.deck.json', '').replace('.json', '') + '.pdf'); out.parent.mkdir(parents=True, exist_ok=True)
|
|
315
|
+
b.page.pdf(path=str(out), width='1280px', height='720px', print_background=True, margin={'top': '0', 'right': '0', 'bottom': '0', 'left': '0'}, page_ranges='')
|
|
316
|
+
result['pdf'] = str(out.relative_to(Path(repo).resolve()))
|
|
317
|
+
if b.errors: result['render_errors'] = b.errors[:3]
|
|
318
|
+
errors = sum(1 for x in issues if x['severity'] == 'error'); warns = sum(1 for x in issues if x['severity'] == 'warn')
|
|
319
|
+
result.update(issues=issues[:60], summary='%d errors, %d warnings, %d notes' % (errors, warns, len(issues) - errors - warns), fixed=changes,
|
|
320
|
+
contact_sheet=str(sheet.relative_to(Path(root))) if sheet else None,
|
|
321
|
+
note=('Fix every error, then run deck_check again. ' if errors else '') + 'The contact sheet of the slides is attached: look at it for anything the checks cannot see (weak hierarchy, empty areas, images that do not fit the story).')
|
|
322
|
+
seen = 'Slides of %s%s' % (rel, ' (slides %s)' % ', '.join(map(str, shown)) if shown else '')
|
|
323
|
+
return result, {'path': str(sheet), 'name': sheet.name, 'note': seen, 'text': seen, 'label': 'Checked the slides of ' + rel}
|
|
324
|
+
|
|
325
|
+
|
|
326
|
+
# ---------- PowerPoint and spreadsheet import ----------
|
|
327
|
+
|
|
328
|
+
NS = {'a': 'http://schemas.openxmlformats.org/drawingml/2006/main', 'p': 'http://schemas.openxmlformats.org/presentationml/2006/main',
|
|
329
|
+
'r': 'http://schemas.openxmlformats.org/officeDocument/2006/relationships', 'c': 'http://schemas.openxmlformats.org/drawingml/2006/chart',
|
|
330
|
+
'rel': 'http://schemas.openxmlformats.org/package/2006/relationships', 's': 'http://schemas.openxmlformats.org/spreadsheetml/2006/main'}
|
|
331
|
+
R_ID = '{%s}id' % NS['r']; R_EMBED = '{%s}embed' % NS['r']
|
|
332
|
+
|
|
333
|
+
def _xml(z, name):
|
|
334
|
+
try: return ET.fromstring(z.read(name))
|
|
335
|
+
except KeyError: return None
|
|
336
|
+
|
|
337
|
+
def _rels(z, part):
|
|
338
|
+
folder, base = part.rsplit('/', 1)
|
|
339
|
+
root = _xml(z, '%s/_rels/%s.rels' % (folder, base))
|
|
340
|
+
out = {}
|
|
341
|
+
if root is None: return out
|
|
342
|
+
for r in root.findall('rel:Relationship', NS):
|
|
343
|
+
target = r.get('Target', '')
|
|
344
|
+
if r.get('TargetMode') == 'External': continue
|
|
345
|
+
parts = folder.split('/')
|
|
346
|
+
for seg in target.split('/'):
|
|
347
|
+
if seg == '..': parts = parts[:-1]
|
|
348
|
+
elif seg and seg != '.': parts.append(seg)
|
|
349
|
+
out[r.get('Id')] = '/'.join(parts) if not target.startswith('/') else target.lstrip('/')
|
|
350
|
+
return out
|
|
351
|
+
|
|
352
|
+
def _color(el):
|
|
353
|
+
if el is None: return None
|
|
354
|
+
s = el.find('a:srgbClr', NS)
|
|
355
|
+
if s is not None: return '#' + s.get('val', '000000').lower()
|
|
356
|
+
s = el.find('a:sysClr', NS)
|
|
357
|
+
if s is not None: return '#' + (s.get('lastClr') or ('000000' if s.get('val') == 'windowText' else 'ffffff')).lower()
|
|
358
|
+
return None
|
|
359
|
+
|
|
360
|
+
def _lum(h):
|
|
361
|
+
h = h.lstrip('#'); r, g, b = (int(h[i:i + 2], 16) / 255 for i in (0, 2, 4))
|
|
362
|
+
return .2126 * r + .7152 * g + .0722 * b
|
|
363
|
+
|
|
364
|
+
def _mix(a, b, k):
|
|
365
|
+
a, b = a.lstrip('#'), b.lstrip('#')
|
|
366
|
+
return '#' + ''.join('%02x' % round(int(a[i:i + 2], 16) + (int(b[i:i + 2], 16) - int(a[i:i + 2], 16)) * k) for i in (0, 2, 4))
|
|
367
|
+
|
|
368
|
+
def _texts(sp):
|
|
369
|
+
"""Paragraph texts of a shape's text body."""
|
|
370
|
+
out = []
|
|
371
|
+
for p in sp.findall('.//a:p', NS):
|
|
372
|
+
t = ''.join(x.text or '' for x in p.findall('.//a:t', NS)).strip()
|
|
373
|
+
if t: out.append(t)
|
|
374
|
+
return out
|
|
375
|
+
|
|
376
|
+
def _chart(z, part):
|
|
377
|
+
root = _xml(z, part)
|
|
378
|
+
if root is None: return None
|
|
379
|
+
plot = root.find('.//c:plotArea', NS)
|
|
380
|
+
if plot is None: return None
|
|
381
|
+
kinds = {'barChart': 'bar', 'bar3DChart': 'bar', 'lineChart': 'line', 'line3DChart': 'line', 'areaChart': 'area', 'pieChart': 'donut', 'doughnutChart': 'donut', 'pie3DChart': 'donut', 'scatterChart': 'scatter'}
|
|
382
|
+
for tag, kind in kinds.items():
|
|
383
|
+
el = plot.find('c:' + tag, NS)
|
|
384
|
+
if el is None: continue
|
|
385
|
+
if kind == 'bar':
|
|
386
|
+
d = el.find('c:barDir', NS); g = el.find('c:grouping', NS)
|
|
387
|
+
if g is not None and g.get('val') in ('stacked', 'percentStacked'): kind = 'stacked'
|
|
388
|
+
elif d is not None and d.get('val') == 'bar': kind = 'hbar'
|
|
389
|
+
labels, series = [], []
|
|
390
|
+
for ser in el.findall('c:ser', NS)[:6]:
|
|
391
|
+
name = ''.join(x.text or '' for x in ser.findall('c:tx//c:v', NS))[:60]
|
|
392
|
+
cats = [x.text or '' for x in ser.findall('c:cat//c:pt/c:v', NS)]
|
|
393
|
+
vals = []
|
|
394
|
+
for x in ser.findall('c:val//c:pt/c:v', NS) or ser.findall('c:yVal//c:pt/c:v', NS):
|
|
395
|
+
try: vals.append(float(x.text))
|
|
396
|
+
except (TypeError, ValueError): vals.append(0.0)
|
|
397
|
+
if kind == 'scatter':
|
|
398
|
+
xs = []
|
|
399
|
+
for x in ser.findall('c:xVal//c:pt/c:v', NS):
|
|
400
|
+
try: xs.append(float(x.text))
|
|
401
|
+
except (TypeError, ValueError): xs.append(0.0)
|
|
402
|
+
series.append({'name': name, 'values': [], 'points': [[a, b] for a, b in zip(xs, vals)][:60]}); continue
|
|
403
|
+
if cats and not labels: labels = cats[:24]
|
|
404
|
+
series.append({'name': name or 'Series %d' % (len(series) + 1), 'values': vals[:24]})
|
|
405
|
+
if series: return {'type': kind, 'labels': labels or [str(i + 1) for i in range(len(series[0]['values']))], 'series': series}
|
|
406
|
+
return None
|
|
407
|
+
|
|
408
|
+
def _save_media(z, part, d, stem):
|
|
409
|
+
data = z.read(part)
|
|
410
|
+
kind = image_kind(data)
|
|
411
|
+
if not kind or len(data) > MAX_IMAGE: return None
|
|
412
|
+
out = d / ('%s.%s' % (stem, 'jpg' if kind == 'jpeg' else kind)); k = 2
|
|
413
|
+
while out.exists(): out = d / ('%s-%d.%s' % (stem, k, 'jpg' if kind == 'jpeg' else kind)); k += 1
|
|
414
|
+
out.write_bytes(data); return out
|
|
415
|
+
|
|
416
|
+
def pptx_theme(z, repo, stem):
|
|
417
|
+
pres = _xml(z, 'ppt/presentation.xml')
|
|
418
|
+
sz = pres.find('p:sldSz', NS) if pres is not None else None
|
|
419
|
+
slide_w = int(sz.get('cx')) if sz is not None else 12192000
|
|
420
|
+
prels = _rels(z, 'ppt/presentation.xml')
|
|
421
|
+
master_part = next((v for v in prels.values() if '/slideMasters/' in v), 'ppt/slideMasters/slideMaster1.xml')
|
|
422
|
+
mrels = _rels(z, master_part)
|
|
423
|
+
theme_part = next((v for v in mrels.values() if '/theme/' in v), 'ppt/theme/theme1.xml')
|
|
424
|
+
theme = _xml(z, theme_part)
|
|
425
|
+
scheme, fonts = {}, {}
|
|
426
|
+
if theme is not None:
|
|
427
|
+
cs = theme.find('.//a:clrScheme', NS)
|
|
428
|
+
if cs is not None:
|
|
429
|
+
for child in cs: scheme[child.tag.split('}')[1]] = _color(child)
|
|
430
|
+
for key, tag in (('heading', 'a:majorFont'), ('font', 'a:minorFont')):
|
|
431
|
+
f = theme.find('.//%s/a:latin' % tag, NS)
|
|
432
|
+
if f is not None and f.get('typeface') and not f.get('typeface').startswith('+'): fonts[key] = f.get('typeface')
|
|
433
|
+
master = _xml(z, master_part)
|
|
434
|
+
d = _public(repo, 'template'); texture = logo = None; bg = None
|
|
435
|
+
if master is not None:
|
|
436
|
+
bgpr = master.find('.//p:cSld/p:bg/p:bgPr', NS)
|
|
437
|
+
if bgpr is not None:
|
|
438
|
+
bg = _color(bgpr.find('a:solidFill', NS))
|
|
439
|
+
blip = bgpr.find('.//a:blip', NS)
|
|
440
|
+
if blip is not None and mrels.get(blip.get(R_EMBED)):
|
|
441
|
+
saved = _save_media(z, mrels[blip.get(R_EMBED)], d, stem + '-background')
|
|
442
|
+
if saved: texture = str(saved.relative_to(Path(repo).resolve()))
|
|
443
|
+
ref = master.find('.//p:cSld/p:bg/p:bgRef', NS)
|
|
444
|
+
if ref is not None and bg is None:
|
|
445
|
+
sc = ref.find('a:schemeClr', NS)
|
|
446
|
+
if sc is not None: bg = scheme.get({'bg1': 'lt1', 'bg2': 'lt2', 'tx1': 'dk1', 'tx2': 'dk2'}.get(sc.get('val'), sc.get('val')))
|
|
447
|
+
# A small picture on the master is the logo; a full-bleed one is the background art.
|
|
448
|
+
for pic in master.findall('.//p:pic', NS):
|
|
449
|
+
ext = pic.find('.//a:xfrm/a:ext', NS); blip = pic.find('.//a:blip', NS)
|
|
450
|
+
if ext is None or blip is None or not mrels.get(blip.get(R_EMBED)): continue
|
|
451
|
+
w = int(ext.get('cx', '0'))
|
|
452
|
+
if w < slide_w * .35 and not logo:
|
|
453
|
+
saved = _save_media(z, mrels[blip.get(R_EMBED)], d, stem + '-logo')
|
|
454
|
+
if saved: logo = str(saved.relative_to(Path(repo).resolve()))
|
|
455
|
+
elif w >= slide_w * .9 and not texture:
|
|
456
|
+
saved = _save_media(z, mrels[blip.get(R_EMBED)], d, stem + '-background')
|
|
457
|
+
if saved: texture = str(saved.relative_to(Path(repo).resolve()))
|
|
458
|
+
light_bg, dark_bg = scheme.get('lt1') or '#ffffff', scheme.get('dk1') or '#111111'
|
|
459
|
+
background = bg or light_bg
|
|
460
|
+
dark = _lum(background) < .4
|
|
461
|
+
text = light_bg if dark else dark_bg
|
|
462
|
+
if abs(_lum(text) - _lum(background)) < .4: text = '#ffffff' if dark else '#111111'
|
|
463
|
+
accents = [scheme.get('accent%d' % i) for i in range(1, 7) if scheme.get('accent%d' % i)]
|
|
464
|
+
# The first accent that reads on the background, then the next distinct one.
|
|
465
|
+
usable = [a for a in accents if abs(_lum(a) - _lum(background)) > .18] or accents or ['#4472c4']
|
|
466
|
+
accent = usable[0]; accent2 = next((a for a in usable[1:] if a != accent), accents[1] if len(accents) > 1 else _mix(accent, text, .4))
|
|
467
|
+
out = {'background': background, 'surface': _mix(background, text, .06) if dark else ('#ffffff' if _lum(background) < .98 else _mix(background, text, .03)), 'text': text,
|
|
468
|
+
'muted': _mix(text, background, .38), 'accent': accent, 'accent2': accent2, 'font': fonts.get('font') or 'Inter', 'style': 'plain'}
|
|
469
|
+
if fonts.get('heading') and fonts.get('heading') != out['font']: out['heading'] = fonts['heading']
|
|
470
|
+
if logo: out['logo'] = logo
|
|
471
|
+
if texture: out['texture'] = texture
|
|
472
|
+
return out, {'scheme': scheme, 'fonts': fonts, 'slide_width_emu': slide_w}
|
|
473
|
+
|
|
474
|
+
def pptx_slides(z, repo, stem):
|
|
475
|
+
prels = _rels(z, 'ppt/presentation.xml'); pres = _xml(z, 'ppt/presentation.xml')
|
|
476
|
+
order = [prels.get(s.get(R_ID)) for s in pres.findall('.//p:sldIdLst/p:sldId', NS)] if pres is not None else []
|
|
477
|
+
sz = pres.find('p:sldSz', NS) if pres is not None else None
|
|
478
|
+
slide_w, slide_h = (int(sz.get('cx')), int(sz.get('cy'))) if sz is not None else (12192000, 6858000)
|
|
479
|
+
d = _public(repo, 'imported'); slides = []; logos = {}
|
|
480
|
+
for k, part in enumerate(p for p in order if p):
|
|
481
|
+
root = _xml(z, part)
|
|
482
|
+
if root is None: continue
|
|
483
|
+
rels = _rels(z, part); title = subtitle = kicker = None; body = []; pics = []; bleed = []; table = None; chart = None; loose = []; small = []
|
|
484
|
+
for sp in root.findall('.//p:sp', NS):
|
|
485
|
+
ph = sp.find('.//p:nvPr/p:ph', NS); kind = ph.get('type') if ph is not None else None
|
|
486
|
+
texts = _texts(sp)
|
|
487
|
+
if not texts: continue
|
|
488
|
+
if kind in ('title', 'ctrTitle') and not title: title = ' '.join(texts)
|
|
489
|
+
elif kind == 'subTitle' and not subtitle: subtitle = ' '.join(texts)
|
|
490
|
+
elif kind in ('sldNum', 'dt', 'ftr'): continue
|
|
491
|
+
elif ph is not None: body += texts
|
|
492
|
+
else:
|
|
493
|
+
off = sp.find('.//a:xfrm/a:off', NS); sizes = [int(r.get('sz')) for r in sp.findall('.//a:rPr', NS) if (r.get('sz') or '').isdigit()]
|
|
494
|
+
loose.append((int(off.get('y', 0)) if off is not None else 0, int(off.get('x', 0)) if off is not None else 0, max(sizes) if sizes else 1800, texts))
|
|
495
|
+
# Decks drawn without placeholders: the largest text near the top is the title, a short capitalised line above it the
|
|
496
|
+
# kicker, and the rest is body in reading order.
|
|
497
|
+
loose.sort()
|
|
498
|
+
if loose and not title:
|
|
499
|
+
big = max(loose, key=lambda x: (x[2], -x[0]))
|
|
500
|
+
if big[2] >= 2400 or len(loose) == 1:
|
|
501
|
+
title = ' '.join(big[3]); ty = big[0]; loose.remove(big)
|
|
502
|
+
for x in list(loose):
|
|
503
|
+
t0 = ' '.join(x[3])
|
|
504
|
+
if x[0] < ty and len(t0) < 40 and t0.upper() == t0 and not kicker: kicker = t0; loose.remove(x)
|
|
505
|
+
for x in loose:
|
|
506
|
+
if re.fullmatch(r'\d{1,2}\s*/\s*\d{1,3}', ' '.join(x[3])): continue # page numbers
|
|
507
|
+
body += x[3]
|
|
508
|
+
for pic in root.findall('.//p:pic', NS):
|
|
509
|
+
blip = pic.find('.//a:blip', NS)
|
|
510
|
+
if blip is None or not rels.get(blip.get(R_EMBED)): continue
|
|
511
|
+
ext = pic.find('.//a:xfrm/a:ext', NS)
|
|
512
|
+
cx, cy = (int(ext.get('cx', 0)), int(ext.get('cy', 0))) if ext is not None else (slide_w, slide_h)
|
|
513
|
+
if cx < slide_w * .2 and cy < slide_h * .3: small.append(rels[blip.get(R_EMBED)]); continue # logos and icons, not content
|
|
514
|
+
(bleed if cx >= slide_w * .9 and cy >= slide_h * .9 else pics).append(rels[blip.get(R_EMBED)])
|
|
515
|
+
if not pics and bleed and len(body) <= 1 and k: pics = bleed[-1:] # a full-bleed photo slide
|
|
516
|
+
elif not pics and bleed and k == 0 and len(bleed) > 1: pics = bleed[-1:]
|
|
517
|
+
for media in dict.fromkeys(small): logos[z.read(media)] = logos.get(z.read(media), 0) + 1
|
|
518
|
+
saved_pics = []
|
|
519
|
+
for j, media in enumerate(pics[:4]):
|
|
520
|
+
saved = _save_media(z, media, d, '%s-%02d' % (stem, k + 1))
|
|
521
|
+
if saved: saved_pics.append(str(saved.relative_to(Path(repo).resolve())))
|
|
522
|
+
pics = saved_pics
|
|
523
|
+
tbl = root.find('.//a:tbl', NS)
|
|
524
|
+
if tbl is not None:
|
|
525
|
+
rows = [[' '.join(_texts(tc)) for tc in tr.findall('a:tc', NS)] for tr in tbl.findall('a:tr', NS)]
|
|
526
|
+
if rows: table = {'columns': rows[0][:8], 'rows': [r[:8] for r in rows[1:13]]}
|
|
527
|
+
for gf in root.findall('.//p:graphicFrame', NS):
|
|
528
|
+
c = gf.find('.//c:chart', NS)
|
|
529
|
+
if c is not None and rels.get(c.get(R_ID)): chart = _chart(z, rels[c.get(R_ID)]) or chart
|
|
530
|
+
notes = None
|
|
531
|
+
npart = next((v for v in rels.values() if '/notesSlides/' in v), None)
|
|
532
|
+
if npart:
|
|
533
|
+
nroot = _xml(z, npart)
|
|
534
|
+
if nroot is not None:
|
|
535
|
+
parts = []
|
|
536
|
+
for sp in nroot.findall('.//p:sp', NS):
|
|
537
|
+
ph = sp.find('.//p:nvPr/p:ph', NS)
|
|
538
|
+
if ph is not None and ph.get('type') == 'body': parts += _texts(sp)
|
|
539
|
+
notes = '\n'.join(parts)[:4000] or None
|
|
540
|
+
s = {'title': (title or '')[:200], 'kicker': (kicker or '')[:60] or None}
|
|
541
|
+
if k == 0: s.update(layout='cover', subtitle=(subtitle or ' '.join(body[:1]))[:300] or None, image=pics[0] if pics else None)
|
|
542
|
+
elif chart: s.update(layout='chart', chart=chart, body=' '.join(body)[:400] or None)
|
|
543
|
+
elif table: s.update(layout='table', table=table)
|
|
544
|
+
elif pics and len(pics) >= 3 and not body: s.update(layout='gallery', items=[{'image': p} for p in pics[:4]])
|
|
545
|
+
elif pics and body: s.update(layout='split', bullets=[b[:200] for b in body[:8]], image=pics[0])
|
|
546
|
+
elif pics: s.update(layout='image', image=pics[0])
|
|
547
|
+
elif body and len(body) == 1 and len(body[0]) < 160: s.update(layout='statement', body=body[0])
|
|
548
|
+
elif body: s.update(layout='bullets', bullets=[b[:200] for b in body[:12]])
|
|
549
|
+
else: s.update(layout='section', subtitle=subtitle)
|
|
550
|
+
if notes: s['notes'] = notes
|
|
551
|
+
slides.append({key: v for key, v in s.items() if v not in (None, '', [])})
|
|
552
|
+
for s in slides:
|
|
553
|
+
b = s.get('bullets') or []
|
|
554
|
+
if b and sum(bool(re.fullmatch(r'\d{1,2}', x)) for x in b) * 2 >= len(b) - 1: # numbered lists drawn as separate boxes
|
|
555
|
+
s['bullets'] = [x for x in b if not re.fullmatch(r'\d{1,2}', x)]
|
|
556
|
+
if re.fullmatch(r'(agenda|contents|outline|today)', s.get('title', '').strip().lower()) and s.get('bullets'): s['layout'] = 'agenda'
|
|
557
|
+
last = slides[-1] if len(slides) > 1 else None
|
|
558
|
+
if last and last.get('layout') in ('statement', 'section', 'bullets', 'split') and len(last.get('bullets') or []) <= 2:
|
|
559
|
+
last.update(layout='closing', subtitle=last.get('subtitle') or ' · '.join(last.pop('bullets', []) or []) or last.pop('body', None))
|
|
560
|
+
last.pop('image', None); last.pop('bullets', None)
|
|
561
|
+
if not last.get('subtitle'): last.pop('subtitle', None)
|
|
562
|
+
# A small picture on most slides is the company logo.
|
|
563
|
+
logo = None
|
|
564
|
+
if logos:
|
|
565
|
+
data, n = max(logos.items(), key=lambda x: x[1])
|
|
566
|
+
if n >= max(2, len(slides) // 2) and image_kind(data):
|
|
567
|
+
ext = 'jpg' if image_kind(data) == 'jpeg' else image_kind(data); out = _public(repo, 'template') / ('%s-logo.%s' % (stem, ext))
|
|
568
|
+
if not out.exists(): out.write_bytes(data)
|
|
569
|
+
logo = str(out.relative_to(Path(repo).resolve()))
|
|
570
|
+
return slides, logo
|
|
571
|
+
|
|
572
|
+
def _col(ref):
|
|
573
|
+
m = re.match(r'([A-Z]+)(\d+)$', ref or '')
|
|
574
|
+
if not m: return None
|
|
575
|
+
n = 0
|
|
576
|
+
for ch in m.group(1): n = n * 26 + ord(ch) - 64
|
|
577
|
+
return n - 1, int(m.group(2)) - 1
|
|
578
|
+
|
|
579
|
+
def _range(rows, rng):
|
|
580
|
+
if not rng: return rows
|
|
581
|
+
m = re.fullmatch(r'([A-Z]+\d+):([A-Z]+\d+)', rng.strip().upper())
|
|
582
|
+
if not m: raise ValueError('range looks like A1:D13')
|
|
583
|
+
(c0, r0), (c1, r1) = _col(m.group(1)), _col(m.group(2))
|
|
584
|
+
return [r[c0:c1 + 1] for r in rows[r0:r1 + 1]]
|
|
585
|
+
|
|
586
|
+
def xlsx_rows(data, sheet=None):
|
|
587
|
+
z = zipfile.ZipFile(io.BytesIO(data))
|
|
588
|
+
shared = []
|
|
589
|
+
ss = _xml(z, 'xl/sharedStrings.xml')
|
|
590
|
+
if ss is not None:
|
|
591
|
+
for si in ss.findall('s:si', NS): shared.append(''.join(t.text or '' for t in si.iter('{%s}t' % NS['s'])))
|
|
592
|
+
wb = _xml(z, 'xl/workbook.xml'); rels = _rels(z, 'xl/workbook.xml'); sheets = []
|
|
593
|
+
if wb is not None:
|
|
594
|
+
for sh in wb.findall('s:sheets/s:sheet', NS): sheets.append((sh.get('name'), rels.get(sh.get(R_ID))))
|
|
595
|
+
if not sheets: raise ValueError('No sheets in this workbook')
|
|
596
|
+
names = [n for n, _ in sheets]
|
|
597
|
+
part = next((p for n, p in sheets if n == sheet), None) if sheet else sheets[0][1]
|
|
598
|
+
if not part: raise ValueError('No sheet named %s (sheets: %s)' % (sheet, ', '.join(names)))
|
|
599
|
+
root = _xml(z, part); rows = []
|
|
600
|
+
for row in root.findall('.//s:sheetData/s:row', NS):
|
|
601
|
+
r = int(row.get('r', len(rows) + 1)) - 1
|
|
602
|
+
while len(rows) <= r: rows.append([])
|
|
603
|
+
for c in row.findall('s:c', NS):
|
|
604
|
+
pos = _col(c.get('r', ''))
|
|
605
|
+
col = pos[0] if pos else len(rows[r])
|
|
606
|
+
v = c.find('s:v', NS); t = c.get('t'); val = ''
|
|
607
|
+
if t == 's' and v is not None: val = shared[int(v.text)] if v.text and int(v.text) < len(shared) else ''
|
|
608
|
+
elif t == 'inlineStr': val = ''.join(x.text or '' for x in c.iter('{%s}t' % NS['s']))
|
|
609
|
+
elif v is not None: val = v.text or ''
|
|
610
|
+
while len(rows[r]) <= col: rows[r].append('')
|
|
611
|
+
rows[r][col] = val
|
|
612
|
+
if len(rows) > 2000: break
|
|
613
|
+
return rows, names
|
|
614
|
+
|
|
615
|
+
def _num(v):
|
|
616
|
+
s = str(v).strip().replace(',', '').replace('$', '').replace('€', '').replace('£', '').replace('%', '')
|
|
617
|
+
try: return float(s)
|
|
618
|
+
except ValueError: return None
|
|
619
|
+
|
|
620
|
+
def table_to_chart(rows):
|
|
621
|
+
rows = [r for r in rows if any(str(c).strip() for c in r)]
|
|
622
|
+
if len(rows) < 2: raise ValueError('Need a header row and at least one data row')
|
|
623
|
+
width = max(len(r) for r in rows); rows = [list(r) + [''] * (width - len(r)) for r in rows]
|
|
624
|
+
head, data = rows[0], rows[1:]
|
|
625
|
+
numeric = [j for j in range(1, width) if sum(_num(r[j]) is not None for r in data) >= max(1, len(data) * .7)]
|
|
626
|
+
if not numeric: raise ValueError('No numeric columns found (the first column is read as labels)')
|
|
627
|
+
unit = '%' if any('%' in str(r[j]) for r in data for j in numeric) else '$' if any('$' in str(r[j]) for r in data for j in numeric) else None
|
|
628
|
+
labels = [str(r[0]).strip()[:40] for r in data][:60]
|
|
629
|
+
series = [{'name': str(head[j]).strip()[:60] or 'Series %d' % (k + 1), 'values': [(_num(r[j]) or 0.0) for r in data][:60]} for k, j in enumerate(numeric[:6])]
|
|
630
|
+
if not unit:
|
|
631
|
+
m = re.search(r'\(([^)]{1,6})\)', str(head[numeric[0]]))
|
|
632
|
+
if m: unit = m.group(1).strip()
|
|
633
|
+
big = max((abs(v) for s in series for v in s['values']), default=0)
|
|
634
|
+
peaks = [max((abs(v) for v in s['values']), default=0) for s in series]
|
|
635
|
+
mixed = len(series) > 1 and min(peaks) > 0 and max(peaks) / min(peaks) > 8
|
|
636
|
+
hint = 'line' if len(labels) > 8 else 'bar'
|
|
637
|
+
if len(series) == 1 and len(labels) <= 6 and unit == '%' and abs(sum(series[0]['values']) - 100) < 2: hint = 'donut'
|
|
638
|
+
return {'chart': {'type': hint, 'labels': labels, 'series': series, **({'unit': unit} if unit else {})}, 'rows': len(data), 'columns': [str(h) for h in head][:12],
|
|
639
|
+
'note': ('Use chart on a chart slide. Pick the type for the point: bar or hbar to compare, line for trends, stacked for parts of a whole over time, waterfall for a bridge, funnel for conversion.'
|
|
640
|
+
+ (' The series are on very different scales (%s); chart one per slide, or the small one disappears.' % ', '.join(s['name'] for s in series) if mixed else '')
|
|
641
|
+
+ (' Values are large; scale them and set a unit such as $M.' if big >= 1e6 else ''))}
|
|
642
|
+
|
|
643
|
+
def deck_import(args, root, repo):
|
|
644
|
+
rel = str(args.get('path') or '').strip()
|
|
645
|
+
path = _inside(repo, rel)
|
|
646
|
+
if not path.is_file(): raise ValueError('File not found: %s' % rel)
|
|
647
|
+
if path.stat().st_size > MAX_IMPORT: raise ValueError('File too large to import')
|
|
648
|
+
_count(root, 'deck_import')
|
|
649
|
+
ext = path.suffix.lower(); stem = _stem(path.stem)
|
|
650
|
+
if ext == '.csv':
|
|
651
|
+
rows = list(csv.reader(io.StringIO(path.read_text(errors='replace'))))
|
|
652
|
+
return table_to_chart(_range(rows, args.get('range')))
|
|
653
|
+
if ext in ('.xlsx', '.xlsm'):
|
|
654
|
+
rows, names = xlsx_rows(path.read_bytes(), args.get('sheet'))
|
|
655
|
+
out = table_to_chart(_range(rows, args.get('range'))); out['sheets'] = names; return out
|
|
656
|
+
if ext not in ('.pptx', '.potx', '.ppsx'): raise ValueError('Import a .pptx template or deck, or an .xlsx or .csv spreadsheet')
|
|
657
|
+
try: z = zipfile.ZipFile(path)
|
|
658
|
+
except zipfile.BadZipFile: raise ValueError('Not a valid PowerPoint file') from None
|
|
659
|
+
if sum(i.file_size for i in z.infolist()) > MAX_IMPORT * 3: raise ValueError('File expands too large to import')
|
|
660
|
+
theme, raw = pptx_theme(z, repo, stem)
|
|
661
|
+
note = 'Use this as the deck theme (keep logo and texture paths). '
|
|
662
|
+
if theme['font'] not in WEB_FONTS: note += '"%s" is not a web font: slides fall back to a similar system font on screen, and PowerPoint uses the real font. ' % theme['font']
|
|
663
|
+
out = {'theme': theme, 'template': raw}
|
|
664
|
+
if (args.get('mode') or 'theme') == 'deck':
|
|
665
|
+
slides, logo = pptx_slides(z, repo, stem)
|
|
666
|
+
if logo and not theme.get('logo'): theme['logo'] = logo
|
|
667
|
+
if not slides: raise ValueError('No slides found in this file')
|
|
668
|
+
target = _inside(repo, 'decks/%s.deck.json' % stem); k = 2
|
|
669
|
+
while target.exists(): target = _inside(repo, 'decks/%s-%d.deck.json' % (stem, k)); k += 1
|
|
670
|
+
target.parent.mkdir(parents=True, exist_ok=True)
|
|
671
|
+
title = slides[0].get('title') or path.stem
|
|
672
|
+
target.write_text(json.dumps({'version': 1, 'title': title, 'theme': theme, 'slides': slides}, indent=2) + '\n')
|
|
673
|
+
out.update(deck=str(target.relative_to(Path(repo).resolve())), slides=len(slides))
|
|
674
|
+
note += 'Converted %d slides. Review the layouts (rewrite topic titles as takeaways, turn dense bullets into charts, stats or diagrams), then run deck_check.' % len(slides)
|
|
675
|
+
out['note'] = note
|
|
676
|
+
return out
|
|
677
|
+
|
|
678
|
+
WEB_FONTS = {'Inter', 'Inter Tight', 'Sora', 'Archivo', 'Archivo Black', 'Fraunces', 'Source Serif 4', 'Space Grotesk', 'IBM Plex Sans', 'IBM Plex Mono', 'JetBrains Mono', 'DM Sans', 'DM Serif Display',
|
|
679
|
+
'Bricolage Grotesque', 'Manrope', 'Instrument Serif', 'Plus Jakarta Sans', 'Syne', 'Playfair Display', 'Outfit', 'Poppins', 'Montserrat', 'Lora', 'Space Mono', 'Libre Baskerville',
|
|
680
|
+
'Cormorant Garamond', 'Unbounded', 'Figtree', 'Geist'}
|
|
681
|
+
|
|
682
|
+
|
|
683
|
+
# ---------- free images and icons ----------
|
|
684
|
+
|
|
685
|
+
def _strip(html):
|
|
686
|
+
return re.sub(r'\s+', ' ', re.sub(r'<[^>]+>', '', str(html or ''))).strip()[:120]
|
|
687
|
+
|
|
688
|
+
def search_openverse(query):
|
|
689
|
+
q = urllib.parse.urlencode({'q': query, 'license_type': 'commercial', 'page_size': 12, 'mature': 'false'})
|
|
690
|
+
data = json.loads(_get(OPENVERSE + '?' + q, accept='application/json'))
|
|
691
|
+
out = []
|
|
692
|
+
for h in data.get('results', [])[:12]:
|
|
693
|
+
lic = ('cc0' if h.get('license') == 'cc0' else 'Public domain' if h.get('license') == 'pdm' else 'CC %s %s' % (str(h.get('license', '')).upper(), h.get('license_version') or '')).strip()
|
|
694
|
+
if str(h.get('license', '')).lower() not in ('cc0', 'pdm', 'by', 'by-sa'): continue
|
|
695
|
+
if not str(h.get('url', '')).startswith('https://'): continue
|
|
696
|
+
out.append({'id': 'ov:' + str(h.get('id')), 'title': str(h.get('title') or '')[:100], 'url': h['url'], 'width': h.get('width') or 0, 'height': h.get('height') or 0,
|
|
697
|
+
'creator': str(h.get('creator') or '')[:80], 'license': lic.replace('cc0', 'CC0'), 'page': str(h.get('foreign_landing_url') or '')[:300], 'via': 'Openverse'})
|
|
698
|
+
return out
|
|
699
|
+
|
|
700
|
+
def search_commons(query):
|
|
701
|
+
q = urllib.parse.urlencode({'action': 'query', 'format': 'json', 'generator': 'search', 'gsrnamespace': 6, 'gsrsearch': query + ' filetype:bitmap', 'gsrlimit': 12,
|
|
702
|
+
'prop': 'imageinfo', 'iiprop': 'url|size|mime|extmetadata', 'iiurlwidth': 1600, 'iiextmetadatafilter': 'Artist|LicenseShortName|LicenseUrl|UsageTerms|AttributionRequired|ObjectName'})
|
|
703
|
+
data = json.loads(_get(COMMONS + '?' + q, accept='application/json'))
|
|
704
|
+
out = []
|
|
705
|
+
for page in sorted((data.get('query') or {}).get('pages', {}).values(), key=lambda p: p.get('index', 0)):
|
|
706
|
+
info = (page.get('imageinfo') or [{}])[0]; meta = info.get('extmetadata') or {}
|
|
707
|
+
lic = _strip((meta.get('LicenseShortName') or {}).get('value'))
|
|
708
|
+
if not OPEN_LICENSES.match(lic) or re.search(r'\bNC\b|\bND\b', lic, re.I): continue
|
|
709
|
+
if info.get('mime') not in ('image/jpeg', 'image/png', 'image/webp'): continue
|
|
710
|
+
url = info.get('thumburl') or info.get('url')
|
|
711
|
+
if not str(url).startswith('https://'): continue
|
|
712
|
+
out.append({'id': 'wm:' + str(page.get('pageid')), 'title': _strip((meta.get('ObjectName') or {}).get('value')) or str(page.get('title', '')).removeprefix('File:').rsplit('.', 1)[0][:100],
|
|
713
|
+
'url': url, 'width': info.get('thumbwidth') or info.get('width') or 0, 'height': info.get('thumbheight') or info.get('height') or 0,
|
|
714
|
+
'creator': _strip((meta.get('Artist') or {}).get('value')) or 'Wikimedia Commons contributor', 'license': lic, 'page': str(info.get('descriptionurl') or '')[:300], 'via': 'Wikimedia Commons'})
|
|
715
|
+
return out
|
|
716
|
+
|
|
717
|
+
def credit_line(h):
|
|
718
|
+
who = h['creator'] or 'Unknown'
|
|
719
|
+
if h['license'] in ('CC0', 'Public domain'): return '%s, %s (%s)' % (h['title'] or 'Image', h['license'], h['via'])
|
|
720
|
+
return '"%s" by %s, %s (%s)' % (h['title'] or 'Image', who, h['license'], h['via'])
|
|
721
|
+
|
|
722
|
+
def find_image(args, root, repo):
|
|
723
|
+
_offline()
|
|
724
|
+
query = ' '.join(str(args.get('query') or '').split())[:100]
|
|
725
|
+
if not query: raise ValueError('query is required')
|
|
726
|
+
stem = _stem(args.get('filename')); source = args.get('source') or 'auto'; pick = str(args.get('pick') or '')
|
|
727
|
+
d = _public(repo, 'stock')
|
|
728
|
+
if any(d.glob(stem + '.*')): raise ValueError('Asset filename already exists. Choose a new name.')
|
|
729
|
+
_count(root, 'find_image')
|
|
730
|
+
hits = []
|
|
731
|
+
for name, fn in (('openverse', search_openverse), ('wikimedia', search_commons)):
|
|
732
|
+
if source in ('auto', name):
|
|
733
|
+
try: hits += fn(query)
|
|
734
|
+
except Exception as exc:
|
|
735
|
+
if source == name: raise ValueError('%s search failed: %s' % (name, str(exc)[:200])) from None
|
|
736
|
+
# Prefer large landscape images for slides.
|
|
737
|
+
hits.sort(key=lambda h: (-(min(h['width'], 2400) * (1.2 if h['width'] >= h['height'] else .7))))
|
|
738
|
+
if not hits: return {'saved': None, 'note': 'No freely licensed images matched. Try simpler words, or stock_photo.', 'matches': []}
|
|
739
|
+
chosen = next((h for h in hits if pick and h['id'] == pick), hits[0])
|
|
740
|
+
data = _get(chosen['url'], limit=MAX_IMAGE, timeout=60)
|
|
741
|
+
kind = image_kind(data)
|
|
742
|
+
if kind not in ('jpeg', 'png', 'webp'): raise ValueError('The download was not a photo')
|
|
743
|
+
path = d / ('%s.%s' % (stem, 'jpg' if kind == 'jpeg' else kind))
|
|
744
|
+
with path.open('xb') as f: f.write(data)
|
|
745
|
+
credit = credit_line(chosen)
|
|
746
|
+
with (d / 'CREDITS.md').open('a') as f: f.write('- %s: %s %s\n' % (path.name, credit, chosen['page']))
|
|
747
|
+
rel = str(path.relative_to(Path(repo).resolve()))
|
|
748
|
+
return {'saved': {'path': rel, 'width': chosen['width'], 'height': chosen['height'], 'credit': credit, 'source': chosen['page']},
|
|
749
|
+
'note': 'Set the slide\'s credit field to the credit line (required by the licence) and add the source page to its sources if you cite the image.',
|
|
750
|
+
'matches': [{k: h[k] for k in ('id', 'title', 'width', 'height', 'license', 'via')} for h in hits if h is not chosen][:8]}
|
|
751
|
+
|
|
752
|
+
def find_icon(args, root, repo):
|
|
753
|
+
_offline()
|
|
754
|
+
name = str(args.get('name') or '').strip().lower(); query = ' '.join(str(args.get('query') or '').split())[:60]
|
|
755
|
+
color = str(args.get('color') or '').strip()
|
|
756
|
+
if color and not re.fullmatch(r'#?[0-9a-fA-F]{6}', color): raise ValueError('color is a hex colour such as #8b5cf6')
|
|
757
|
+
if not name and not query: raise ValueError('Give query (e.g. "rocket") or name (e.g. "simple-icons:stripe")')
|
|
758
|
+
_count(root, 'find_icon')
|
|
759
|
+
others = []
|
|
760
|
+
if not name:
|
|
761
|
+
q = urllib.parse.urlencode({'query': query, 'limit': 32, 'prefixes': ','.join(ICON_SETS)})
|
|
762
|
+
found = json.loads(_get(ICONIFY + '/search?' + q, accept='application/json')).get('icons') or []
|
|
763
|
+
# Line icon sets first for concepts; brand sets first when the query looks like a company or product.
|
|
764
|
+
rank = {p: i for i, p in enumerate(ICON_SETS)}
|
|
765
|
+
found.sort(key=lambda n: rank.get(n.split(':')[0], 99))
|
|
766
|
+
slug = re.sub(r'[^a-z0-9]+', '-', query.lower()).strip('-')
|
|
767
|
+
generic = [n for n in found if n.split(':')[0] not in ('simple-icons', 'logos') and n.split(':')[1] == slug]
|
|
768
|
+
brand = [n for n in found if n in ('logos:' + slug, 'logos:' + slug + '-icon', 'simple-icons:' + slug.replace('-', ''))]
|
|
769
|
+
if not generic and brand: # a company or product name: its logo (full colour unless a colour was asked for)
|
|
770
|
+
brand.sort(key=lambda n: n.startswith('simple-icons') != bool(color)); found = brand + [n for n in found if n not in brand]
|
|
771
|
+
elif generic: found = generic + [n for n in found if n not in generic]
|
|
772
|
+
if not found: return {'saved': None, 'note': 'No icons matched. Try one plain word (rocket, shield, chart).', 'matches': []}
|
|
773
|
+
name = found[0]; others = found[1:10]
|
|
774
|
+
if not re.fullmatch(r'[a-z0-9-]+:[a-z0-9-]+', name): raise ValueError('name looks like set:icon, e.g. lucide:rocket')
|
|
775
|
+
prefix, icon = name.split(':')
|
|
776
|
+
if not color and prefix != 'logos':
|
|
777
|
+
raise ValueError('%s is a one-colour icon and draws in black unless you give a colour. Call again with name="%s" and color set to the deck accent (theme.accent).' % (name, name))
|
|
778
|
+
params = {'height': '64'}
|
|
779
|
+
if color: params['color'] = '#' + color.lstrip('#')
|
|
780
|
+
svg = _get('%s/%s/%s.svg?%s' % (ICONIFY, prefix, icon, urllib.parse.urlencode(params)), limit=200_000, accept='image/svg+xml')
|
|
781
|
+
if b'<svg' not in svg[:400].lower() or b'<script' in svg.lower() or b'foreignobject' in svg.lower(): raise ValueError('Icon not found: %s' % name)
|
|
782
|
+
d = _public(repo, 'icons'); path = d / ('%s-%s%s.svg' % (prefix, icon, '-' + color.lstrip('#').lower() if color else ''))
|
|
783
|
+
path.write_bytes(svg)
|
|
784
|
+
return {'saved': {'path': str(path.relative_to(Path(repo).resolve())), 'name': name}, 'matches': others,
|
|
785
|
+
'note': 'Use the path as an item icon, a diagram node icon or a logos-slide image. Brand logos are trademarks: use them to refer to the company, not as your own branding.'}
|
|
786
|
+
|
|
787
|
+
|
|
788
|
+
# ---------- image preparation ----------
|
|
789
|
+
|
|
790
|
+
ASPECTS = {'16:9': 16 / 9, '4:3': 4 / 3, '1:1': 1.0, '3:4': 3 / 4, '9:16': 9 / 16, '3:2': 1.5, '2:3': 2 / 3}
|
|
791
|
+
|
|
792
|
+
def image_prep(args, root, repo, ensure_browser=lambda: False):
|
|
793
|
+
rel = str(args.get('path') or '').strip(); op = args.get('op')
|
|
794
|
+
src = _inside(repo, rel)
|
|
795
|
+
if not src.is_file(): raise ValueError('Image not found: %s' % rel)
|
|
796
|
+
if op not in ('cutout', 'crop', 'focus', 'darken', 'mono', 'duotone'): raise ValueError('op is cutout, crop, focus, darken, mono or duotone')
|
|
797
|
+
data = src.read_bytes()
|
|
798
|
+
if image_kind(data) not in ('jpeg', 'png', 'webp', 'gif'): raise ValueError('Give a photo (jpg, png or webp)')
|
|
799
|
+
aspect = args.get('aspect')
|
|
800
|
+
if op == 'crop' and aspect not in ASPECTS: raise ValueError('aspect is one of ' + ', '.join(ASPECTS))
|
|
801
|
+
for key in ('dark', 'bright'):
|
|
802
|
+
if args.get(key) and not re.fullmatch(r'#[0-9a-fA-F]{6}', str(args[key])): raise ValueError('%s is a hex colour such as #0b0b10' % key)
|
|
803
|
+
if not ensure_browser(): raise ValueError('Image preparation needs the test browser, which is not available here. Use the image as is, or set imageFocus/imageTreatment on the slide (the slide renders those itself).')
|
|
804
|
+
_count(root, 'image_prep')
|
|
805
|
+
with Browser(root) as b:
|
|
806
|
+
res = b.page.evaluate('([u, op, o]) => window.__imagePrep(u, op, o)', [data_url(data), op, {k: v for k, v in {'aspect': ASPECTS.get(aspect), 'dark': args.get('dark'), 'bright': args.get('bright')}.items() if v}])
|
|
807
|
+
if not res.get('ok'): return {'saved': None, 'note': res.get('note')}
|
|
808
|
+
if op == 'focus': return {'focus': res['focus'], 'note': res['note']}
|
|
809
|
+
raw = base64.b64decode(res['url'].split(',', 1)[1]); ext = '.png' if op == 'cutout' else '.jpg'
|
|
810
|
+
out = src.with_name('%s-%s%s' % (src.stem, op if op != 'crop' else 'crop-' + aspect.replace(':', 'x'), ext)); k = 2
|
|
811
|
+
while out.exists(): out = src.with_name('%s-%s-%d%s' % (src.stem, op, k, ext)); k += 1
|
|
812
|
+
out.write_bytes(raw)
|
|
813
|
+
return {'saved': {'path': str(out.relative_to(Path(repo).resolve())), 'width': res.get('width'), 'height': res.get('height')}, 'note': res['note'] + ' The original is unchanged.'}
|
|
814
|
+
|
|
815
|
+
|
|
816
|
+
def call(name, args, root, repo, ensure_browser=None):
|
|
817
|
+
"""Run one deck tool. Returns (result, screenshot or None); the screenshot is shown to the model."""
|
|
818
|
+
if name not in NAMES: raise ValueError('Unknown deck tool')
|
|
819
|
+
if not isinstance(args, dict): raise ValueError('Arguments must be an object')
|
|
820
|
+
ensure = ensure_browser or (lambda: browser_ready(root))
|
|
821
|
+
if name == 'deck_check': return deck_check(args, root, repo, ensure)
|
|
822
|
+
if name == 'deck_import': return deck_import(args, root, repo), None
|
|
823
|
+
if name == 'find_image': return find_image(args, root, repo), None
|
|
824
|
+
if name == 'find_icon': return find_icon(args, root, repo), None
|
|
825
|
+
return image_prep(args, root, repo, ensure), None
|