unaltraweb 0.3.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +7 -0
- data/LICENSE +20 -0
- data/Makefile +412 -0
- data/README.md +226 -0
- data/_config.yml +842 -0
- data/_data/i18n/ca.yml +278 -0
- data/_data/i18n/en.yml +278 -0
- data/_data/i18n/es.yml +278 -0
- data/_includes/audio.liquid +30 -0
- data/_includes/bib_search.liquid +5 -0
- data/_includes/blog-list.liquid +66 -0
- data/_includes/book-citation.liquid +33 -0
- data/_includes/citation.liquid +26 -0
- data/_includes/custom_head.liquid +3 -0
- data/_includes/custom_scripts.liquid +5 -0
- data/_includes/cv/list.liquid +5 -0
- data/_includes/cv/list_groups.liquid +51 -0
- data/_includes/cv/map.liquid +29 -0
- data/_includes/cv/nested_list.liquid +16 -0
- data/_includes/cv/time_table.liquid +118 -0
- data/_includes/cv-download-card.liquid +26 -0
- data/_includes/developer-mode-switcher.liquid +24 -0
- data/_includes/disqus.liquid +13 -0
- data/_includes/distill_scripts.liquid +298 -0
- data/_includes/documentation-download-detail.liquid +76 -0
- data/_includes/documentation-downloads.liquid +71 -0
- data/_includes/documentation-profile-attributes.liquid +4 -0
- data/_includes/documentation-section-toc.liquid +9 -0
- data/_includes/documentation-sidebar.liquid +112 -0
- data/_includes/figure.liquid +87 -0
- data/_includes/footer.liquid +66 -0
- data/_includes/giscus.liquid +25 -0
- data/_includes/head.liquid +220 -0
- data/_includes/header.liquid +544 -0
- data/_includes/latest_posts.liquid +61 -0
- data/_includes/localized-date.liquid +5 -0
- data/_includes/manual-bibliography.liquid +2 -0
- data/_includes/manual-featured-readings.liquid +12 -0
- data/_includes/manual-other-readings.liquid +12 -0
- data/_includes/manual-page-toc.liquid +11 -0
- data/_includes/manual-sidebar.liquid +58 -0
- data/_includes/manual-teachers.liquid +74 -0
- data/_includes/metadata.liquid +278 -0
- data/_includes/news.liquid +48 -0
- data/_includes/newsletter.liquid +68 -0
- data/_includes/page-hero.liquid +96 -0
- data/_includes/page-link.liquid +8 -0
- data/_includes/pagination.liquid +22 -0
- data/_includes/post-reading-time.liquid +29 -0
- data/_includes/previous-next.liquid +37 -0
- data/_includes/profile-card.liquid +49 -0
- data/_includes/profile-feature-matrix.liquid +75 -0
- data/_includes/profile-highlights.liquid +122 -0
- data/_includes/project-grid.liquid +26 -0
- data/_includes/project-resource-link.liquid +45 -0
- data/_includes/project-resources.liquid +21 -0
- data/_includes/projects.liquid +38 -0
- data/_includes/projects_horizontal.liquid +37 -0
- data/_includes/publications-metrics-summary.liquid +69 -0
- data/_includes/publications-summary.liquid +127 -0
- data/_includes/reading-biblio-controls.liquid +65 -0
- data/_includes/reading-cover-card.liquid +93 -0
- data/_includes/reading-rating.liquid +11 -0
- data/_includes/recommended-readings.liquid +30 -0
- data/_includes/related_posts.liquid +39 -0
- data/_includes/repository/repo.liquid +48 -0
- data/_includes/repository/repo_trophies.liquid +44 -0
- data/_includes/repository/repo_user.liquid +35 -0
- data/_includes/resume/awards.liquid +19 -0
- data/_includes/resume/basics.liquid +28 -0
- data/_includes/resume/certificates.liquid +35 -0
- data/_includes/resume/education.liquid +54 -0
- data/_includes/resume/interests.liquid +34 -0
- data/_includes/resume/languages.liquid +31 -0
- data/_includes/resume/projects.liquid +32 -0
- data/_includes/resume/publications.liquid +28 -0
- data/_includes/resume/references.liquid +31 -0
- data/_includes/resume/skills.liquid +33 -0
- data/_includes/resume/volunteer.liquid +52 -0
- data/_includes/resume/work.liquid +52 -0
- data/_includes/scripts.liquid +364 -0
- data/_includes/search-data.js.liquid +341 -0
- data/_includes/selected_papers.liquid +3 -0
- data/_includes/t.liquid +31 -0
- data/_includes/team-grid.liquid +172 -0
- data/_includes/video.liquid +97 -0
- data/_layouts/about.liquid +93 -0
- data/_layouts/archive.liquid +45 -0
- data/_layouts/bib.liquid +677 -0
- data/_layouts/book-review.liquid +225 -0
- data/_layouts/book-shelf.liquid +66 -0
- data/_layouts/cv.liquid +127 -0
- data/_layouts/default.liquid +70 -0
- data/_layouts/distill.liquid +132 -0
- data/_layouts/documentation-download-detail.liquid +5 -0
- data/_layouts/documentation-home.liquid +100 -0
- data/_layouts/documentation-page.liquid +45 -0
- data/_layouts/manual-bib.liquid +73 -0
- data/_layouts/manual-chapter.liquid +99 -0
- data/_layouts/manual-featured-bib.liquid +100 -0
- data/_layouts/manual-home.liquid +120 -0
- data/_layouts/none.liquid +1 -0
- data/_layouts/outputs.liquid +29 -0
- data/_layouts/page.liquid +52 -0
- data/_layouts/post.liquid +95 -0
- data/_layouts/profile.liquid +36 -0
- data/_layouts/profiles.liquid +36 -0
- data/_layouts/project.liquid +7 -0
- data/_layouts/repositories.liquid +58 -0
- data/_layouts/theses.liquid +126 -0
- data/_plugins/bibliography_profiles.rb +164 -0
- data/_plugins/callouts.rb +104 -0
- data/_plugins/code_blocks.rb +277 -0
- data/_plugins/computation_figure_images.rb +266 -0
- data/_plugins/computation_sources.rb +42 -0
- data/_plugins/content_search_index.rb +210 -0
- data/_plugins/details.rb +23 -0
- data/_plugins/external-posts.rb +124 -0
- data/_plugins/figure_captions.rb +800 -0
- data/_plugins/file-exists.rb +22 -0
- data/_plugins/google-scholar-citations.rb +86 -0
- data/_plugins/hide-custom-bibtex.rb +18 -0
- data/_plugins/inspirehep-citations.rb +57 -0
- data/_plugins/localized_visual_sources.rb +145 -0
- data/_plugins/manual_release_metadata.rb +47 -0
- data/_plugins/mermaid_mmd_images.rb +233 -0
- data/_plugins/profile-pages.rb +28 -0
- data/_plugins/remove-accents.rb +32 -0
- data/_plugins/reproducible_build_time.rb +55 -0
- data/_plugins/search-data.rb +83 -0
- data/_plugins/theme-cache-bust.rb +56 -0
- data/_plugins/vega_visual_images.rb +290 -0
- data/_plugins/web_capture_images.rb +154 -0
- data/_sass/_base.scss +2133 -0
- data/_sass/_blog.scss +67 -0
- data/_sass/_callouts.scss +100 -0
- data/_sass/_custom.scss +437 -0
- data/_sass/_cv.scss +274 -0
- data/_sass/_developer.scss +97 -0
- data/_sass/_distill.scss +185 -0
- data/_sass/_documentation.scss +2220 -0
- data/_sass/_layout.scss +218 -0
- data/_sass/_manual.scss +2691 -0
- data/_sass/_profile.scss +323 -0
- data/_sass/_projects.scss +163 -0
- data/_sass/_reading.scss +465 -0
- data/_sass/_site-custom.scss +3 -0
- data/_sass/_tabs.scss +48 -0
- data/_sass/_team.scss +230 -0
- data/_sass/_themes.scss +254 -0
- data/_sass/_typograms.scss +132 -0
- data/_sass/_variables.scss +52 -0
- data/_sass/font-awesome/_animated.scss +152 -0
- data/_sass/font-awesome/_bordered-pulled.scss +20 -0
- data/_sass/font-awesome/_core.scss +49 -0
- data/_sass/font-awesome/_fixed-width.scss +7 -0
- data/_sass/font-awesome/_functions.scss +57 -0
- data/_sass/font-awesome/_icons.scss +13 -0
- data/_sass/font-awesome/_list.scss +18 -0
- data/_sass/font-awesome/_mixins.scss +65 -0
- data/_sass/font-awesome/_rotated-flipped.scss +31 -0
- data/_sass/font-awesome/_screen-reader.scss +14 -0
- data/_sass/font-awesome/_shims.scss +1578 -0
- data/_sass/font-awesome/_sizing.scss +16 -0
- data/_sass/font-awesome/_stacked.scss +32 -0
- data/_sass/font-awesome/_variables.scss +5044 -0
- data/_sass/font-awesome/brands.scss +30 -0
- data/_sass/font-awesome/fontawesome.scss +21 -0
- data/_sass/font-awesome/regular.scss +27 -0
- data/_sass/font-awesome/solid.scss +27 -0
- data/_sass/font-awesome/v4-shims.scss +11 -0
- data/_sass/tabler-icons/tabler-icons-filled.scss +1646 -0
- data/_sass/tabler-icons/tabler-icons-outline.scss +9859 -0
- data/_sass/tabler-icons/tabler-icons.scss +11463 -0
- data/_scripts/cronitor-analytics-setup.js +9 -0
- data/_scripts/giscus-setup.js +48 -0
- data/_scripts/google-analytics-setup.js +9 -0
- data/_scripts/open-panel-analytics-setup.js +14 -0
- data/_scripts/photoswipe-setup.js +11 -0
- data/assets/audio/epicaly-short-113909.mp3 +0 -0
- data/assets/bibliography/2018-12-22-distill.bib +7 -0
- data/assets/css/academicons.min.css +1 -0
- data/assets/css/bootstrap-toc.min.css +4 -0
- data/assets/css/bootstrap.min.css +7 -0
- data/assets/css/bootstrap.min.css.map +1 -0
- data/assets/css/jekyll-pygments-themes-github.css +209 -0
- data/assets/css/jekyll-pygments-themes-native.css +232 -0
- data/assets/css/jupyter-grade3.css +3758 -0
- data/assets/css/jupyter-monokai.css +3758 -0
- data/assets/css/jupyter.css +383 -0
- data/assets/css/main.scss +36 -0
- data/assets/css/mdb.min.css +48 -0
- data/assets/css/mdb.min.css.map +1 -0
- data/assets/css/scholar-icons.css +386 -0
- data/assets/fonts/academicons.ttf +0 -0
- data/assets/fonts/academicons.woff +0 -0
- data/assets/fonts/scholar-icons.ttf +0 -0
- data/assets/fonts/scholar-icons.woff +0 -0
- data/assets/fonts/tabler-icons-filled.ttf +0 -0
- data/assets/fonts/tabler-icons-filled.woff +0 -0
- data/assets/fonts/tabler-icons-filled.woff2 +0 -0
- data/assets/fonts/tabler-icons-outline.ttf +0 -0
- data/assets/fonts/tabler-icons-outline.woff +0 -0
- data/assets/fonts/tabler-icons-outline.woff2 +0 -0
- data/assets/fonts/tabler-icons.ttf +0 -0
- data/assets/fonts/tabler-icons.woff +0 -0
- data/assets/fonts/tabler-icons.woff2 +0 -0
- data/assets/html/relativity.html +15 -0
- data/assets/img/1.jpg +0 -0
- data/assets/img/10.jpg +0 -0
- data/assets/img/11.jpg +0 -0
- data/assets/img/12.jpg +0 -0
- data/assets/img/2.jpg +0 -0
- data/assets/img/3.jpg +0 -0
- data/assets/img/4.jpg +0 -0
- data/assets/img/5.jpg +0 -0
- data/assets/img/6.jpg +0 -0
- data/assets/img/7.jpg +0 -0
- data/assets/img/8.jpg +0 -0
- data/assets/img/9.jpg +0 -0
- data/assets/img/book_covers/the_godfather.jpg +0 -0
- data/assets/img/brand/dosquartsdedocs-logo-cafe.svg +31 -0
- data/assets/img/brand/dosquartsdedocs-logo-footer-cafe.svg +31 -0
- data/assets/img/brand/dosquartsdedocs-logo-white.svg +31 -0
- data/assets/img/brand/dosquartsdedocs-logo.svg +31 -0
- data/assets/img/prof_pic.jpg +0 -0
- data/assets/img/prof_pic_color.png +0 -0
- data/assets/img/profile-placeholder.svg +8 -0
- data/assets/img/publication_preview/brownian-motion.gif +0 -0
- data/assets/img/publication_preview/goodchild1992-gis-cover.jpg +0 -0
- data/assets/img/publication_preview/gutierrez2020-jtrg-preview.jpg +0 -0
- data/assets/img/publication_preview/harvey1989-postmodernity-cover.jpg +0 -0
- data/assets/img/publication_preview/tobler1970-economic-geography-cover.jpg +0 -0
- data/assets/img/publication_preview/tuan1977-space-place-cover.jpg +0 -0
- data/assets/img/publication_preview/wave-mechanics.gif +0 -0
- data/assets/img/publication_preview/zaragozi2025-rooftop-pv-preview.jpg +0 -0
- data/assets/img/rhino.png +0 -0
- data/assets/img/team/aaron-gutierrez.png +0 -0
- data/assets/img/team/alba-font.png +0 -0
- data/assets/img/team/alba-viana.png +0 -0
- data/assets/img/team/alejandro-gonzalez.png +0 -0
- data/assets/img/team/ana-pastor.png +0 -0
- data/assets/img/team/antoni-domenech.png +0 -0
- data/assets/img/team/benito-zaragozi.png +0 -0
- data/assets/img/team/cinta-sanz.png +0 -0
- data/assets/img/team/daniel-miravet.png +0 -0
- data/assets/img/team/eugenia-altamirano.png +0 -0
- data/assets/img/team/fiammetta-brandajs.png +0 -0
- data/assets/img/team/francesc-gonzalez.png +0 -0
- data/assets/img/team/isabel-paulino.png +0 -0
- data/assets/img/team/joan-borras.png +0 -0
- data/assets/img/team/julie-wilson.png +0 -0
- data/assets/img/team/liliana-sole.png +0 -0
- data/assets/img/team/maria-casado.png +0 -0
- data/assets/img/team/maria-medina.png +0 -0
- data/assets/img/team/marta-nello.png +0 -0
- data/assets/img/team/mogrify +1 -0
- data/assets/img/team/oscar-saladie.png +0 -0
- data/assets/img/team/paolo-russo.png +0 -0
- data/assets/img/team/raquel-santos.png +0 -0
- data/assets/img/team/salvador-anton.png +0 -0
- data/assets/img/team/sunsi-huertas.png +0 -0
- data/assets/img/team/user-placeholder.png +0 -0
- data/assets/img/team/wilbert-den-hoed.png +0 -0
- data/assets/img/team/xavier-delclos.png +0 -0
- data/assets/img/template_error.png +0 -0
- data/assets/js/bibsearch.js +70 -0
- data/assets/js/bootstrap-toc.min.js +5 -0
- data/assets/js/bootstrap.bundle.min.js +7 -0
- data/assets/js/bootstrap.bundle.min.js.map +1 -0
- data/assets/js/callouts.js +115 -0
- data/assets/js/chartjs-setup.js +14 -0
- data/assets/js/common.js +117 -0
- data/assets/js/content-search-match.js +159 -0
- data/assets/js/content-search.js +441 -0
- data/assets/js/copy_code.js +60 -0
- data/assets/js/custom.js +6 -0
- data/assets/js/developer-mode.js +103 -0
- data/assets/js/diff2html-setup.js +20 -0
- data/assets/js/distillpub/overrides.js +27 -0
- data/assets/js/distillpub/template.v2.js +9616 -0
- data/assets/js/distillpub/template.v2.js.map +1 -0
- data/assets/js/distillpub/transforms.v2.js +14557 -0
- data/assets/js/distillpub/transforms.v2.js.map +1 -0
- data/assets/js/documentation.js +435 -0
- data/assets/js/echarts-setup.js +29 -0
- data/assets/js/highlight-search-term.js +110 -0
- data/assets/js/jupyter_new_tab.js +18 -0
- data/assets/js/leaflet-setup.js +22 -0
- data/assets/js/manual.js +308 -0
- data/assets/js/masonry.js +12 -0
- data/assets/js/mathjax-setup.js +46 -0
- data/assets/js/mermaid-setup.js +37 -0
- data/assets/js/newsletter.js +105 -0
- data/assets/js/no_defer.js +24 -0
- data/assets/js/plotly-setup.js +52 -0
- data/assets/js/profile-nav.js +92 -0
- data/assets/js/progress-bar.js +73 -0
- data/assets/js/pseudocode-setup.js +18 -0
- data/assets/js/search/base-styles.min.js +286 -0
- data/assets/js/search/command-score.min.js +7 -0
- data/assets/js/search/hotkeys-js/hotkeys.esm.min.js +18 -0
- data/assets/js/search/lit/decorators.min.js +7 -0
- data/assets/js/search/lit/directives/class-map.min.js +7 -0
- data/assets/js/search/lit/directives/join.min.js +7 -0
- data/assets/js/search/lit/directives/live.min.js +7 -0
- data/assets/js/search/lit/directives/ref.min.js +7 -0
- data/assets/js/search/lit/directives/repeat.min.js +7 -0
- data/assets/js/search/lit/directives/unsafe-html.min.js +7 -0
- data/assets/js/search/lit/index.min.js +7 -0
- data/assets/js/search/lit-element/lit-element.min.js +12 -0
- data/assets/js/search/lit-html/directives/class-map.min.js +22 -0
- data/assets/js/search/lit-html/directives/join.min.js +12 -0
- data/assets/js/search/lit-html/directives/live.min.js +28 -0
- data/assets/js/search/lit-html/directives/ref.min.js +32 -0
- data/assets/js/search/lit-html/directives/repeat.min.js +27 -0
- data/assets/js/search/lit-html/directives/unsafe-html.min.js +22 -0
- data/assets/js/search/lit-html/is-server.min.js +12 -0
- data/assets/js/search/lit-html/lit-html.min.js +12 -0
- data/assets/js/search/ninja-action.min.js +101 -0
- data/assets/js/search/ninja-footer.min.js +62 -0
- data/assets/js/search/ninja-header.min.js +78 -0
- data/assets/js/search/ninja-keys.min.js +39 -0
- data/assets/js/search/reactive-element/decorators/custom-element.min.js +12 -0
- data/assets/js/search/reactive-element/decorators/event-options.min.js +12 -0
- data/assets/js/search/reactive-element/decorators/property.min.js +22 -0
- data/assets/js/search/reactive-element/decorators/query-all.min.js +17 -0
- data/assets/js/search/reactive-element/decorators/query-assigned-elements.min.js +17 -0
- data/assets/js/search/reactive-element/decorators/query-assigned-nodes.min.js +17 -0
- data/assets/js/search/reactive-element/decorators/query-async.min.js +17 -0
- data/assets/js/search/reactive-element/decorators/query.min.js +17 -0
- data/assets/js/search/reactive-element/decorators/state.min.js +27 -0
- data/assets/js/search/reactive-element/reactive-element.min.js +17 -0
- data/assets/js/search-setup.js +17 -0
- data/assets/js/shortcut-key.js +11 -0
- data/assets/js/tabs.min.js +5 -0
- data/assets/js/theme.js +314 -0
- data/assets/js/tooltips-setup.js +3 -0
- data/assets/js/typograms.js +1341 -0
- data/assets/js/vanilla-back-to-top.min.js +1 -0
- data/assets/js/vega-setup.js +24 -0
- data/assets/js/venobox-setup.js +5 -0
- data/assets/js/wechat.js +14 -0
- data/assets/js/zoom.js +15 -0
- data/assets/json/resume.json +98 -0
- data/assets/json/table_data.json +128 -0
- data/assets/jupyter/blog.ipynb +48 -0
- data/assets/pdf/example_pdf.pdf +0 -0
- data/assets/plotly/demo.html +72 -0
- data/assets/video/pexels-engin-akyurt-6069112-960x540-30fps.mp4 +0 -0
- data/assets/webfonts/fa-brands-400.ttf +0 -0
- data/assets/webfonts/fa-brands-400.woff2 +0 -0
- data/assets/webfonts/fa-regular-400.ttf +0 -0
- data/assets/webfonts/fa-regular-400.woff2 +0 -0
- data/assets/webfonts/fa-solid-900.ttf +0 -0
- data/assets/webfonts/fa-solid-900.woff2 +0 -0
- data/assets/webfonts/fa-v4compatibility.ttf +0 -0
- data/assets/webfonts/fa-v4compatibility.woff2 +0 -0
- data/docs/Gemfile +5 -0
- data/docs/_config.yml +83 -0
- data/docs/_documentation/en/01-quick-start.md +115 -0
- data/docs/_documentation/en/02-tools.md +146 -0
- data/docs/_documentation/en/03-usage.md +107 -0
- data/docs/_documentation/en/04-jekyll-basics.md +51 -0
- data/docs/_documentation/en/05-pages-collections.md +68 -0
- data/docs/_documentation/en/06-github-web-editing.md +92 -0
- data/docs/_documentation/en/10-profiles.md +125 -0
- data/docs/_documentation/en/11-unaltreselfie.md +39 -0
- data/docs/_documentation/en/12-unaltreprojecte.md +38 -0
- data/docs/_documentation/en/13-unaltremanual.md +237 -0
- data/docs/_documentation/en/14-unaltredocs.md +92 -0
- data/docs/_documentation/en/15-profile-checklist.md +39 -0
- data/docs/_documentation/en/20-syntax.md +353 -0
- data/docs/_documentation/en/21-themes.md +103 -0
- data/docs/_documentation/en/22-feature-reference.md +56 -0
- data/docs/_documentation/en/23-front-matter.md +51 -0
- data/docs/_documentation/en/24-data-files.md +28 -0
- data/docs/_documentation/en/30-customization.md +448 -0
- data/docs/_documentation/en/31-template.md +55 -0
- data/docs/_documentation/en/32-development.md +108 -0
- data/docs/_documentation/en/33-reusable-workflows.md +136 -0
- data/docs/_documentation/en/40-distribution.md +164 -0
- data/docs/_documentation/en/41-bibliometrics.md +137 -0
- data/docs/_documentation/en/42-docker-image.md +185 -0
- data/docs/_documentation/en/50-faq.md +33 -0
- data/docs/_pages/ca/index.md +35 -0
- data/docs/_pages/en/index.md +35 -0
- data/docs/_pages/es/index.md +35 -0
- data/docs/agents/action-prompts/00-start-site-session.txt +10 -0
- data/docs/agents/action-prompts/05-create-new-web.txt +5 -0
- data/docs/agents/action-prompts/10-content-update.txt +9 -0
- data/docs/agents/action-prompts/15-edit-default-content.txt +8 -0
- data/docs/agents/action-prompts/20-manual-teaching-materials.txt +18 -0
- data/docs/agents/action-prompts/22-manual-style-audit.txt +14 -0
- data/docs/agents/action-prompts/23-manual-structure-audit.txt +17 -0
- data/docs/agents/action-prompts/25-translation-prepublish.txt +8 -0
- data/docs/agents/action-prompts/30-project-site-update.txt +7 -0
- data/docs/agents/action-prompts/40-documentation-update.txt +7 -0
- data/docs/agents/action-prompts/50-bibliography-entry.txt +7 -0
- data/docs/agents/action-prompts/60-bibliometrics-refresh.txt +7 -0
- data/docs/agents/action-prompts/70-build-and-review.txt +12 -0
- data/docs/agents/manual-authoring-components.md +255 -0
- data/docs/agents/mcp-contract.md +162 -0
- data/docs/agents/quarto-computation-adoption.md +102 -0
- data/docs/assets/img/unaltraweb-docs-hero.svg +36 -0
- data/lib/unaltraweb/version.rb +8 -0
- data/lib/unaltraweb.rb +31 -0
- data/requirements.txt +5 -0
- data/scripts/biblio/fetch_scimago_csv.sh +126 -0
- data/scripts/biblio/metrics_common.py +154 -0
- data/scripts/biblio/metrics_merge_meta.py +73 -0
- data/scripts/biblio/metrics_update.py +826 -0
- data/scripts/computations/python/Dockerfile +37 -0
- data/scripts/computations/r/Dockerfile +31 -0
- data/scripts/computations/render.py +1267 -0
- data/scripts/cv/render_pdf_preview.sh +36 -0
- data/scripts/deploy/publish_branch.sh +80 -0
- data/scripts/generate_logo_variants.py +47 -0
- data/scripts/manual/Dockerfile +31 -0
- data/scripts/manual/build_pdf.py +1780 -0
- data/scripts/manual/filters/bibliography.lua +164 -0
- data/scripts/manual/filters/code-blocks.lua +102 -0
- data/scripts/manual/filters/figure-captions.lua +35 -0
- data/scripts/manual/publish_release.sh +281 -0
- data/scripts/manual/templates/manual.tex +494 -0
- data/scripts/manual/verify_release_assets.py +650 -0
- data/scripts/test_gem_build.py +128 -0
- data/scripts/test_reproducible_jekyll_build.py +191 -0
- data/scripts/test_wheel_install.py +282 -0
- data/scripts/unaltraweb-docker-mount.sh +36 -0
- data/scripts/unaltraweb-mcp-bootstrap.sh +105 -0
- data/scripts/unaltraweb-mcp-cleanup.sh +73 -0
- data/scripts/unaltraweb-mcp-project-id.sh +34 -0
- data/scripts/validate_distribution.py +563 -0
- data/scripts/validate_workflows.py +1285 -0
- data/scripts/web_captures/Dockerfile +31 -0
- data/scripts/web_captures/capture.mjs +157 -0
- data/scripts/web_captures/package-lock.json +56 -0
- data/scripts/web_captures/package.json +9 -0
- data/scripts/web_captures/render.py +867 -0
- data/src/unaltraweb_mcp/component-contract.json +153 -0
- data/src/unaltraweb_mcp/component-contract.schema.json +87 -0
- data/src/unaltraweb_mcp/docker_mount.py +19 -0
- metadata +861 -0
|
@@ -0,0 +1,1267 @@
|
|
|
1
|
+
#!/usr/bin/env python3
|
|
2
|
+
"""Render versioned executable chapter sources to Markdown and figures."""
|
|
3
|
+
|
|
4
|
+
from __future__ import annotations
|
|
5
|
+
|
|
6
|
+
import argparse
|
|
7
|
+
import contextlib
|
|
8
|
+
import datetime as dt
|
|
9
|
+
import fcntl
|
|
10
|
+
import hashlib
|
|
11
|
+
import html
|
|
12
|
+
import json
|
|
13
|
+
import os
|
|
14
|
+
import re
|
|
15
|
+
import secrets
|
|
16
|
+
import shutil
|
|
17
|
+
import subprocess
|
|
18
|
+
import sys
|
|
19
|
+
import tempfile
|
|
20
|
+
from pathlib import Path
|
|
21
|
+
from typing import Any
|
|
22
|
+
|
|
23
|
+
try:
|
|
24
|
+
import yaml
|
|
25
|
+
except ImportError: # pragma: no cover - optional for status/check on lean hosts
|
|
26
|
+
yaml = None
|
|
27
|
+
|
|
28
|
+
|
|
29
|
+
CONFIG_PATH = Path(".unaltraweb/computations.yml")
|
|
30
|
+
LOCK_PATH = Path(".unaltraweb/computations.lock.json")
|
|
31
|
+
SUPPORTED_SUFFIXES = {".qmd", ".rmd", ".r", ".py", ".ipynb"}
|
|
32
|
+
GENERATED_MEDIA_SUFFIXES = {".gif", ".jpeg", ".jpg", ".pdf", ".png", ".svg", ".webp"}
|
|
33
|
+
DEFAULT_IMAGES = {
|
|
34
|
+
"python": "ghcr.io/dosquartsdedocs/unaltraweb-compute-python@sha256:18cb269811bd4005800382da25a480ec2bca7eac8d0501ad1ef36bad1c0f8cd9",
|
|
35
|
+
"r": "ghcr.io/dosquartsdedocs/unaltraweb-compute-r@sha256:928ffb93f221e09e8b929157dee473b838e061915a2eb67224e4124b85f81837",
|
|
36
|
+
}
|
|
37
|
+
ENGINE_ENV = {"python": "COMPUTE_PYTHON_IMAGE", "r": "COMPUTE_R_IMAGE"}
|
|
38
|
+
FRONT_MATTER_RE = re.compile(r"\A---\s*\n(.*?)\n---\s*\n?", re.DOTALL)
|
|
39
|
+
IMAGE_RE = re.compile(
|
|
40
|
+
r"!\[([^\]]*)\]\(([^)\s]+)(?:\s+[\"']([^\"']*)[\"'])?\)(\{[^}\n]*\})?"
|
|
41
|
+
)
|
|
42
|
+
QUARTO_FIGURE_RE = re.compile(r'<div\s+id="fig-[^"]+"[^>]*>\s*<img\s+(.*?)\s*/>\s*(.*?)\s*</div>', re.DOTALL | re.IGNORECASE)
|
|
43
|
+
|
|
44
|
+
|
|
45
|
+
class ComputationError(RuntimeError):
|
|
46
|
+
pass
|
|
47
|
+
|
|
48
|
+
|
|
49
|
+
def strip_inline_comment(value: str) -> str:
|
|
50
|
+
in_single = False
|
|
51
|
+
in_double = False
|
|
52
|
+
for index, char in enumerate(value):
|
|
53
|
+
if char == "'" and not in_double:
|
|
54
|
+
in_single = not in_single
|
|
55
|
+
elif char == '"' and not in_single:
|
|
56
|
+
in_double = not in_double
|
|
57
|
+
elif char == "#" and not in_single and not in_double and (index == 0 or value[index - 1].isspace()):
|
|
58
|
+
return value[:index].strip()
|
|
59
|
+
return value.strip()
|
|
60
|
+
|
|
61
|
+
|
|
62
|
+
def parse_scalar(value: str) -> Any:
|
|
63
|
+
value = strip_inline_comment(value)
|
|
64
|
+
if value in {"", "null", "~"}:
|
|
65
|
+
return None
|
|
66
|
+
if value in {"true", "True"}:
|
|
67
|
+
return True
|
|
68
|
+
if value in {"false", "False"}:
|
|
69
|
+
return False
|
|
70
|
+
if re.fullmatch(r"-?\d+", value):
|
|
71
|
+
return int(value)
|
|
72
|
+
if len(value) >= 2 and value[0] == "[" and value[-1] == "]":
|
|
73
|
+
inner = value[1:-1].strip()
|
|
74
|
+
return [] if not inner else [parse_scalar(part.strip()) for part in inner.split(",")]
|
|
75
|
+
if len(value) >= 2 and value[0] == value[-1] and value[0] in {"'", '"'}:
|
|
76
|
+
return value[1:-1]
|
|
77
|
+
return value
|
|
78
|
+
|
|
79
|
+
|
|
80
|
+
def simple_yaml_load(text: str) -> Any:
|
|
81
|
+
lines = [(len(raw) - len(raw.lstrip(" ")), raw.strip()) for raw in text.splitlines() if raw.strip() and not raw.lstrip().startswith("#")]
|
|
82
|
+
|
|
83
|
+
def parse_block(index: int, indent: int) -> tuple[Any, int]:
|
|
84
|
+
if index >= len(lines):
|
|
85
|
+
return {}, index
|
|
86
|
+
if lines[index][0] < indent:
|
|
87
|
+
return {}, index
|
|
88
|
+
if lines[index][1].startswith("- "):
|
|
89
|
+
items: list[Any] = []
|
|
90
|
+
while index < len(lines) and lines[index][0] == indent and lines[index][1].startswith("- "):
|
|
91
|
+
item = lines[index][1][2:].strip()
|
|
92
|
+
if item:
|
|
93
|
+
items.append(parse_scalar(item))
|
|
94
|
+
index += 1
|
|
95
|
+
else:
|
|
96
|
+
child, index = parse_block(index + 1, indent + 2)
|
|
97
|
+
items.append(child)
|
|
98
|
+
return items, index
|
|
99
|
+
data: dict[str, Any] = {}
|
|
100
|
+
while index < len(lines) and lines[index][0] == indent and not lines[index][1].startswith("- "):
|
|
101
|
+
line = lines[index][1]
|
|
102
|
+
if ":" not in line:
|
|
103
|
+
index += 1
|
|
104
|
+
continue
|
|
105
|
+
key, value = line.split(":", 1)
|
|
106
|
+
key = key.strip()
|
|
107
|
+
value = value.strip()
|
|
108
|
+
if value:
|
|
109
|
+
data[key] = parse_scalar(value)
|
|
110
|
+
index += 1
|
|
111
|
+
else:
|
|
112
|
+
child_indent = lines[index + 1][0] if index + 1 < len(lines) else indent + 2
|
|
113
|
+
child, index = parse_block(index + 1, child_indent)
|
|
114
|
+
data[key] = child
|
|
115
|
+
return data, index
|
|
116
|
+
|
|
117
|
+
parsed, _ = parse_block(0, lines[0][0] if lines else 0)
|
|
118
|
+
return parsed
|
|
119
|
+
|
|
120
|
+
|
|
121
|
+
def simple_yaml_dump(value: Any, indent: int = 0) -> str:
|
|
122
|
+
prefix = " " * indent
|
|
123
|
+
if isinstance(value, dict):
|
|
124
|
+
lines: list[str] = []
|
|
125
|
+
for key, item in value.items():
|
|
126
|
+
if isinstance(item, (dict, list)):
|
|
127
|
+
lines.append(f"{prefix}{key}:")
|
|
128
|
+
lines.append(simple_yaml_dump(item, indent + 2))
|
|
129
|
+
else:
|
|
130
|
+
lines.append(f"{prefix}{key}: {format_scalar(item)}")
|
|
131
|
+
return "\n".join(lines)
|
|
132
|
+
if isinstance(value, list):
|
|
133
|
+
lines = []
|
|
134
|
+
for item in value:
|
|
135
|
+
if isinstance(item, (dict, list)):
|
|
136
|
+
lines.append(f"{prefix}-")
|
|
137
|
+
lines.append(simple_yaml_dump(item, indent + 2))
|
|
138
|
+
else:
|
|
139
|
+
lines.append(f"{prefix}- {format_scalar(item)}")
|
|
140
|
+
return "\n".join(lines)
|
|
141
|
+
return f"{prefix}{format_scalar(value)}"
|
|
142
|
+
|
|
143
|
+
|
|
144
|
+
def format_scalar(value: Any) -> str:
|
|
145
|
+
if value is True:
|
|
146
|
+
return "true"
|
|
147
|
+
if value is False:
|
|
148
|
+
return "false"
|
|
149
|
+
if value is None:
|
|
150
|
+
return "null"
|
|
151
|
+
text = str(value)
|
|
152
|
+
if not text or re.search(r"[:#\[\]{},]|^[-?]|\s$|^\s", text):
|
|
153
|
+
return json.dumps(text, ensure_ascii=False)
|
|
154
|
+
return text
|
|
155
|
+
|
|
156
|
+
|
|
157
|
+
def utc_now() -> str:
|
|
158
|
+
return dt.datetime.now(dt.timezone.utc).replace(microsecond=0).isoformat()
|
|
159
|
+
|
|
160
|
+
|
|
161
|
+
def json_dump(value: Any) -> str:
|
|
162
|
+
return json.dumps(value, ensure_ascii=False, indent=2, sort_keys=True) + "\n"
|
|
163
|
+
|
|
164
|
+
|
|
165
|
+
def safe_relative(project: Path, raw: str, *, label: str, must_exist: bool = False) -> Path:
|
|
166
|
+
value = str(raw or "").strip()
|
|
167
|
+
if not value:
|
|
168
|
+
raise ComputationError(f"No {label} path is configured.")
|
|
169
|
+
relative = Path(value)
|
|
170
|
+
if relative.is_absolute() or ".." in relative.parts:
|
|
171
|
+
raise ComputationError(f"{label} must be a project-relative path: {raw}")
|
|
172
|
+
resolved = (project / relative).resolve()
|
|
173
|
+
try:
|
|
174
|
+
resolved.relative_to(project)
|
|
175
|
+
except ValueError as exc:
|
|
176
|
+
raise ComputationError(f"{label} escapes the project: {raw}") from exc
|
|
177
|
+
if must_exist and not resolved.exists():
|
|
178
|
+
raise ComputationError(f"Missing {label}: {relative}")
|
|
179
|
+
return resolved
|
|
180
|
+
|
|
181
|
+
|
|
182
|
+
def relative(project: Path, path: Path) -> str:
|
|
183
|
+
return str(path.resolve().relative_to(project)).replace(os.sep, "/")
|
|
184
|
+
|
|
185
|
+
|
|
186
|
+
def contains(root: Path, path: Path) -> bool:
|
|
187
|
+
try:
|
|
188
|
+
path.relative_to(root)
|
|
189
|
+
except ValueError:
|
|
190
|
+
return False
|
|
191
|
+
return True
|
|
192
|
+
|
|
193
|
+
|
|
194
|
+
def paths_overlap(left: Path, right: Path) -> bool:
|
|
195
|
+
return contains(left, right) or contains(right, left)
|
|
196
|
+
|
|
197
|
+
|
|
198
|
+
def read_yaml_mapping(text: str, *, label: str) -> dict[str, Any]:
|
|
199
|
+
if yaml is not None:
|
|
200
|
+
try:
|
|
201
|
+
parsed = yaml.safe_load(text)
|
|
202
|
+
except yaml.YAMLError as exc:
|
|
203
|
+
raise ComputationError(f"Invalid YAML in {label}: {exc}") from exc
|
|
204
|
+
else:
|
|
205
|
+
parsed = simple_yaml_load(text)
|
|
206
|
+
if parsed is None:
|
|
207
|
+
return {}
|
|
208
|
+
if not isinstance(parsed, dict):
|
|
209
|
+
raise ComputationError(f"{label} must contain a YAML mapping.")
|
|
210
|
+
return parsed
|
|
211
|
+
|
|
212
|
+
|
|
213
|
+
def comment_content(line: str, prefix: str) -> str:
|
|
214
|
+
value = line[len(prefix):]
|
|
215
|
+
return value[1:] if value.startswith(" ") else value
|
|
216
|
+
|
|
217
|
+
|
|
218
|
+
def front_matter_text(path: Path) -> str:
|
|
219
|
+
suffix = path.suffix.lower()
|
|
220
|
+
if suffix in {".qmd", ".rmd"}:
|
|
221
|
+
text = path.read_text(encoding="utf-8")
|
|
222
|
+
match = FRONT_MATTER_RE.match(text)
|
|
223
|
+
return match.group(1) if match else ""
|
|
224
|
+
if suffix == ".r":
|
|
225
|
+
lines = path.read_text(encoding="utf-8").splitlines()
|
|
226
|
+
yaml_lines: list[str] = []
|
|
227
|
+
active = False
|
|
228
|
+
for line in lines:
|
|
229
|
+
if not line.startswith("#'"):
|
|
230
|
+
if active:
|
|
231
|
+
break
|
|
232
|
+
continue
|
|
233
|
+
value = comment_content(line, "#'")
|
|
234
|
+
if value == "---":
|
|
235
|
+
if active:
|
|
236
|
+
return "\n".join(yaml_lines)
|
|
237
|
+
active = True
|
|
238
|
+
continue
|
|
239
|
+
if active:
|
|
240
|
+
yaml_lines.append(value)
|
|
241
|
+
return ""
|
|
242
|
+
if suffix == ".py":
|
|
243
|
+
lines = path.read_text(encoding="utf-8").splitlines()
|
|
244
|
+
yaml_lines = []
|
|
245
|
+
active = False
|
|
246
|
+
for line in lines:
|
|
247
|
+
if line.strip() == "# ---":
|
|
248
|
+
if active:
|
|
249
|
+
return "\n".join(yaml_lines)
|
|
250
|
+
active = True
|
|
251
|
+
continue
|
|
252
|
+
if active:
|
|
253
|
+
if not line.startswith("#"):
|
|
254
|
+
break
|
|
255
|
+
yaml_lines.append(comment_content(line, "#"))
|
|
256
|
+
return ""
|
|
257
|
+
if suffix == ".ipynb":
|
|
258
|
+
try:
|
|
259
|
+
notebook = json.loads(path.read_text(encoding="utf-8"))
|
|
260
|
+
except json.JSONDecodeError as exc:
|
|
261
|
+
raise ComputationError(f"Invalid notebook JSON in {path}: {exc}") from exc
|
|
262
|
+
metadata = notebook.get("metadata") if isinstance(notebook, dict) else {}
|
|
263
|
+
if isinstance(metadata, dict) and isinstance(metadata.get("unaltraweb_front_matter"), dict):
|
|
264
|
+
return yaml.safe_dump(metadata["unaltraweb_front_matter"], sort_keys=False) if yaml is not None else simple_yaml_dump(metadata["unaltraweb_front_matter"])
|
|
265
|
+
for cell in notebook.get("cells", []) if isinstance(notebook, dict) else []:
|
|
266
|
+
if not isinstance(cell, dict) or cell.get("cell_type") not in {"raw", "markdown"}:
|
|
267
|
+
continue
|
|
268
|
+
source = cell.get("source", "")
|
|
269
|
+
text = "".join(source) if isinstance(source, list) else str(source)
|
|
270
|
+
match = FRONT_MATTER_RE.match(text)
|
|
271
|
+
if match:
|
|
272
|
+
return match.group(1)
|
|
273
|
+
break
|
|
274
|
+
return ""
|
|
275
|
+
return ""
|
|
276
|
+
|
|
277
|
+
|
|
278
|
+
def read_front_matter(path: Path) -> dict[str, Any]:
|
|
279
|
+
text = front_matter_text(path)
|
|
280
|
+
return read_yaml_mapping(text, label=str(path)) if text else {}
|
|
281
|
+
|
|
282
|
+
|
|
283
|
+
def validate_keys(mapping: dict[str, Any], allowed: set[str], *, label: str) -> None:
|
|
284
|
+
unknown = sorted(str(key) for key in mapping if key not in allowed)
|
|
285
|
+
if unknown:
|
|
286
|
+
raise ComputationError(f"Unknown {label} keys: {', '.join(unknown)}")
|
|
287
|
+
|
|
288
|
+
|
|
289
|
+
def load_config(project: Path) -> tuple[dict[str, Any], Path]:
|
|
290
|
+
path = project / CONFIG_PATH
|
|
291
|
+
if not path.is_file():
|
|
292
|
+
return {
|
|
293
|
+
"version": 1,
|
|
294
|
+
"project_name": project.name,
|
|
295
|
+
"enabled": True,
|
|
296
|
+
"source_roots": ["_chapters"],
|
|
297
|
+
"generated_assets_root": "assets/img/generated",
|
|
298
|
+
"engines": {},
|
|
299
|
+
}, path
|
|
300
|
+
config = read_yaml_mapping(path.read_text(encoding="utf-8"), label=str(CONFIG_PATH))
|
|
301
|
+
validate_keys(
|
|
302
|
+
config,
|
|
303
|
+
{"version", "enabled", "source_roots", "generated_assets_root", "engines"},
|
|
304
|
+
label="computation configuration",
|
|
305
|
+
)
|
|
306
|
+
if config.get("version", 1) != 1:
|
|
307
|
+
raise ComputationError("Unsupported computations.yml version; expected version: 1.")
|
|
308
|
+
roots = config.get("source_roots", ["_chapters"])
|
|
309
|
+
if not isinstance(roots, list) or not roots:
|
|
310
|
+
raise ComputationError("source_roots must be a non-empty list.")
|
|
311
|
+
engines = config.get("engines", {})
|
|
312
|
+
if not isinstance(engines, dict):
|
|
313
|
+
raise ComputationError("engines must be a mapping.")
|
|
314
|
+
for engine, settings in engines.items():
|
|
315
|
+
if engine not in {"r", "python"} or not isinstance(settings, dict):
|
|
316
|
+
raise ComputationError(f"Unsupported computation engine configuration: {engine}")
|
|
317
|
+
validate_keys(
|
|
318
|
+
settings,
|
|
319
|
+
{
|
|
320
|
+
"image",
|
|
321
|
+
"local_image",
|
|
322
|
+
"base_image",
|
|
323
|
+
"environments",
|
|
324
|
+
"dockerfile",
|
|
325
|
+
"context",
|
|
326
|
+
"lockfiles",
|
|
327
|
+
"fingerprint_paths",
|
|
328
|
+
"rstudio",
|
|
329
|
+
},
|
|
330
|
+
label=f"{engine} engine",
|
|
331
|
+
)
|
|
332
|
+
return {
|
|
333
|
+
"version": 1,
|
|
334
|
+
"project_name": project.name,
|
|
335
|
+
"enabled": bool(config.get("enabled", True)),
|
|
336
|
+
"source_roots": [str(item) for item in roots],
|
|
337
|
+
"generated_assets_root": str(config.get("generated_assets_root") or "assets/img/generated"),
|
|
338
|
+
"engines": engines,
|
|
339
|
+
}, path
|
|
340
|
+
|
|
341
|
+
|
|
342
|
+
def compute_environment() -> str:
|
|
343
|
+
explicit = os.environ.get("COMPUTE_ENV", "").strip()
|
|
344
|
+
if explicit:
|
|
345
|
+
return explicit
|
|
346
|
+
return "ci" if os.environ.get("CI", "").lower() in {"1", "true", "yes"} else "local"
|
|
347
|
+
|
|
348
|
+
|
|
349
|
+
def engine_settings(config: dict[str, Any], engine: str) -> dict[str, Any]:
|
|
350
|
+
settings = config.get("engines", {}).get(engine, {})
|
|
351
|
+
return settings if isinstance(settings, dict) else {}
|
|
352
|
+
|
|
353
|
+
|
|
354
|
+
def default_local_image(config: dict[str, Any], engine: str) -> str:
|
|
355
|
+
name = re.sub(r"[^a-z0-9._-]+", "-", str(config["project_name"]).lower()).strip("-._") or "project"
|
|
356
|
+
return f"{name}-compute-{engine}:local"
|
|
357
|
+
|
|
358
|
+
|
|
359
|
+
def resolve_image(config: dict[str, Any], engine: str) -> dict[str, str]:
|
|
360
|
+
if engine not in ENGINE_ENV:
|
|
361
|
+
raise ComputationError(f"Unsupported computation engine: {engine}")
|
|
362
|
+
settings = engine_settings(config, engine)
|
|
363
|
+
env_name = ENGINE_ENV[engine]
|
|
364
|
+
explicit = os.environ.get(env_name, "").strip()
|
|
365
|
+
environment = compute_environment()
|
|
366
|
+
environments = settings.get("environments") if isinstance(settings.get("environments"), dict) else {}
|
|
367
|
+
if explicit:
|
|
368
|
+
image, source = explicit, f"environment:{env_name}"
|
|
369
|
+
elif str(environments.get(environment) or "").strip():
|
|
370
|
+
image, source = str(environments[environment]).strip(), f"config:environments.{environment}"
|
|
371
|
+
elif environment == "local" and settings.get("dockerfile"):
|
|
372
|
+
image = str(settings.get("local_image") or default_local_image(config, engine)).strip()
|
|
373
|
+
source = "config:local_image"
|
|
374
|
+
elif str(settings.get("image") or "").strip():
|
|
375
|
+
image, source = str(settings["image"]).strip(), "config:image"
|
|
376
|
+
else:
|
|
377
|
+
image, source = DEFAULT_IMAGES[engine], "unaltraweb:default"
|
|
378
|
+
if not image:
|
|
379
|
+
raise ComputationError(f"Resolved an empty image for engine {engine}.")
|
|
380
|
+
return {"engine": engine, "environment": environment, "image": image, "source": source}
|
|
381
|
+
|
|
382
|
+
|
|
383
|
+
def source_engine(path: Path, front: dict[str, Any]) -> tuple[str, dict[str, Any]]:
|
|
384
|
+
metadata = front.get("unaltraweb_compute", {})
|
|
385
|
+
if metadata is None:
|
|
386
|
+
metadata = {}
|
|
387
|
+
if not isinstance(metadata, dict):
|
|
388
|
+
raise ComputationError(f"unaltraweb_compute must be a mapping in {path}")
|
|
389
|
+
validate_keys(metadata, {"engine", "inputs", "output", "outputs", "figures", "mode", "enabled"}, label=f"unaltraweb_compute in {path}")
|
|
390
|
+
declared = str(metadata.get("engine") or "").strip().lower()
|
|
391
|
+
suffix = path.suffix.lower()
|
|
392
|
+
inferred = "r" if suffix in {".r", ".rmd"} else "python" if suffix in {".py", ".ipynb"} else ""
|
|
393
|
+
if suffix == ".qmd" and declared not in {"r", "python"}:
|
|
394
|
+
raise ComputationError(f"Quarto source must declare unaltraweb_compute.engine as r or python: {path}")
|
|
395
|
+
engine = declared or inferred
|
|
396
|
+
if engine not in {"r", "python"}:
|
|
397
|
+
raise ComputationError(f"Cannot determine computation engine for {path}")
|
|
398
|
+
if inferred and declared and inferred != declared:
|
|
399
|
+
raise ComputationError(f"Declared engine {declared} conflicts with {path.suffix} source: {path}")
|
|
400
|
+
return engine, metadata
|
|
401
|
+
|
|
402
|
+
|
|
403
|
+
def source_roots(project: Path, config: dict[str, Any]) -> list[Path]:
|
|
404
|
+
roots = [safe_relative(project, item, label="computation source root") for item in config["source_roots"]]
|
|
405
|
+
if any(root == project for root in roots):
|
|
406
|
+
raise ComputationError("A computation source root cannot be the project root.")
|
|
407
|
+
for index, root in enumerate(roots):
|
|
408
|
+
if any(paths_overlap(root, other) for other in roots[index + 1:]):
|
|
409
|
+
raise ComputationError("Computation source roots must not overlap.")
|
|
410
|
+
return roots
|
|
411
|
+
|
|
412
|
+
|
|
413
|
+
def discover_sources(project: Path, config: dict[str, Any], engine_filter: str = "") -> list[dict[str, Any]]:
|
|
414
|
+
if not config["enabled"]:
|
|
415
|
+
return []
|
|
416
|
+
records: list[dict[str, Any]] = []
|
|
417
|
+
roots = source_roots(project, config)
|
|
418
|
+
assets_root = safe_relative(project, config["generated_assets_root"], label="generated assets root")
|
|
419
|
+
if any(paths_overlap(root, assets_root) for root in roots):
|
|
420
|
+
raise ComputationError("generated_assets_root must not overlap a computation source root.")
|
|
421
|
+
for root in roots:
|
|
422
|
+
if not root.is_dir():
|
|
423
|
+
continue
|
|
424
|
+
for path in sorted(item for item in root.rglob("*") if item.is_file() and item.suffix.lower() in SUPPORTED_SUFFIXES):
|
|
425
|
+
front = read_front_matter(path)
|
|
426
|
+
engine, metadata = source_engine(path, front)
|
|
427
|
+
if engine_filter and engine != engine_filter:
|
|
428
|
+
continue
|
|
429
|
+
if metadata.get("enabled") is False:
|
|
430
|
+
continue
|
|
431
|
+
mode = str(metadata.get("mode") or "chapter").strip().lower()
|
|
432
|
+
if mode not in {"chapter", "figure"}:
|
|
433
|
+
raise ComputationError(f"Unsupported unaltraweb_compute.mode in {relative(project, path)}: {mode}")
|
|
434
|
+
if not front.get("title") or not front.get("lang") or not front.get("ref"):
|
|
435
|
+
raise ComputationError(f"Executable source requires title, lang, and ref front matter: {relative(project, path)}")
|
|
436
|
+
source_path = relative(project, path)
|
|
437
|
+
inputs_raw = metadata.get("inputs", [])
|
|
438
|
+
if not isinstance(inputs_raw, list):
|
|
439
|
+
raise ComputationError(f"unaltraweb_compute.inputs must be a list in {source_path}")
|
|
440
|
+
inputs = [safe_relative(project, str(item), label="computation input", must_exist=True) for item in inputs_raw]
|
|
441
|
+
if mode == "figure":
|
|
442
|
+
outputs_raw = metadata.get("outputs")
|
|
443
|
+
if outputs_raw is None and metadata.get("output"):
|
|
444
|
+
outputs_raw = [metadata["output"]]
|
|
445
|
+
if not isinstance(outputs_raw, list) or not outputs_raw:
|
|
446
|
+
raise ComputationError(f"Figure computation requires unaltraweb_compute.outputs in {source_path}")
|
|
447
|
+
outputs = [safe_relative(project, str(item), label="generated figure output") for item in outputs_raw]
|
|
448
|
+
for output in outputs:
|
|
449
|
+
if output.suffix.lower() not in GENERATED_MEDIA_SUFFIXES:
|
|
450
|
+
raise ComputationError(f"Figure computation output must use a supported media suffix: {relative(project, output)}")
|
|
451
|
+
if any(contains(source_root, output) for source_root in roots):
|
|
452
|
+
raise ComputationError(f"Figure computation output must not be inside a configured source root: {relative(project, output)}")
|
|
453
|
+
if len({relative(project, output) for output in outputs}) != len(outputs):
|
|
454
|
+
raise ComputationError(f"Figure computation outputs must be unique in {source_path}")
|
|
455
|
+
for existing in records:
|
|
456
|
+
if existing.get("mode") == "figure":
|
|
457
|
+
if any(output == existing_output for output in outputs for existing_output in existing["outputs"]):
|
|
458
|
+
raise ComputationError(f"Executable sources collide on generated figure outputs: {existing['source_path']} and {source_path}")
|
|
459
|
+
elif any(output == existing["output"] or paths_overlap(output, existing["figures"]) for output in outputs):
|
|
460
|
+
raise ComputationError(f"Generated output paths overlap for {existing['source_path']} and {source_path}")
|
|
461
|
+
records.append(
|
|
462
|
+
{
|
|
463
|
+
"mode": mode,
|
|
464
|
+
"source": path,
|
|
465
|
+
"source_path": source_path,
|
|
466
|
+
"outputs": outputs,
|
|
467
|
+
"output_paths": [relative(project, output) for output in outputs],
|
|
468
|
+
"engine": engine,
|
|
469
|
+
"front": front,
|
|
470
|
+
"inputs": inputs,
|
|
471
|
+
"input_paths": [relative(project, item) for item in inputs],
|
|
472
|
+
}
|
|
473
|
+
)
|
|
474
|
+
continue
|
|
475
|
+
output_raw = str(metadata.get("output") or relative(project, path.with_suffix(".md")))
|
|
476
|
+
output = safe_relative(project, output_raw, label="generated Markdown output")
|
|
477
|
+
if output.suffix.lower() != ".md":
|
|
478
|
+
raise ComputationError(f"Generated chapter output must use .md: {output_raw}")
|
|
479
|
+
if not any(contains(root, output) for root in roots):
|
|
480
|
+
raise ComputationError(f"Generated chapter output must remain under a configured source root: {output_raw}")
|
|
481
|
+
output_path = relative(project, output)
|
|
482
|
+
figures_raw = str(metadata.get("figures") or Path(relative(project, assets_root)) / str(front["lang"]) / str(front["ref"]))
|
|
483
|
+
figures = safe_relative(project, figures_raw, label="generated figures directory")
|
|
484
|
+
if figures == assets_root or not contains(assets_root, figures):
|
|
485
|
+
raise ComputationError(f"Generated figures must use a subdirectory of {relative(project, assets_root)}: {figures_raw}")
|
|
486
|
+
for existing in records:
|
|
487
|
+
if existing.get("mode") == "figure":
|
|
488
|
+
if any(existing_output == output or paths_overlap(existing_output, figures) for existing_output in existing["outputs"]):
|
|
489
|
+
raise ComputationError(f"Generated output paths overlap for {existing['source_path']} and {source_path}")
|
|
490
|
+
continue
|
|
491
|
+
if output == existing["output"]:
|
|
492
|
+
raise ComputationError(f"Executable sources collide on {output_path}: {existing['source_path']} and {source_path}")
|
|
493
|
+
if paths_overlap(figures, existing["figures"]):
|
|
494
|
+
raise ComputationError(f"Executable sources overlap on generated figures: {existing['source_path']} and {source_path}")
|
|
495
|
+
if paths_overlap(output, existing["figures"]) or paths_overlap(figures, existing["output"]):
|
|
496
|
+
raise ComputationError(f"Generated output paths overlap for {existing['source_path']} and {source_path}")
|
|
497
|
+
records.append(
|
|
498
|
+
{
|
|
499
|
+
"mode": mode,
|
|
500
|
+
"source": path,
|
|
501
|
+
"source_path": source_path,
|
|
502
|
+
"output": output,
|
|
503
|
+
"output_path": output_path,
|
|
504
|
+
"figures": figures,
|
|
505
|
+
"figures_path": relative(project, figures),
|
|
506
|
+
"engine": engine,
|
|
507
|
+
"front": front,
|
|
508
|
+
"inputs": inputs,
|
|
509
|
+
"input_paths": [relative(project, item) for item in inputs],
|
|
510
|
+
}
|
|
511
|
+
)
|
|
512
|
+
return records
|
|
513
|
+
|
|
514
|
+
|
|
515
|
+
def path_entries(project: Path, path: Path) -> list[dict[str, Any]]:
|
|
516
|
+
files = [path] if path.is_file() else sorted(item for item in path.rglob("*") if item.is_file())
|
|
517
|
+
if not files:
|
|
518
|
+
raise ComputationError(f"Fingerprint path has no files: {relative(project, path)}")
|
|
519
|
+
return [{"path": relative(project, item), "sha256": hashlib.sha256(item.read_bytes()).hexdigest()} for item in files]
|
|
520
|
+
|
|
521
|
+
|
|
522
|
+
def engine_dependencies(project: Path, config: dict[str, Any], engine: str) -> list[Path]:
|
|
523
|
+
settings = engine_settings(config, engine)
|
|
524
|
+
raw_paths: list[str] = []
|
|
525
|
+
for key in ["dockerfile"]:
|
|
526
|
+
if settings.get(key):
|
|
527
|
+
raw_paths.append(str(settings[key]))
|
|
528
|
+
for key in ["lockfiles", "fingerprint_paths"]:
|
|
529
|
+
values = settings.get(key, [])
|
|
530
|
+
if values and not isinstance(values, list):
|
|
531
|
+
raise ComputationError(f"{engine}.{key} must be a list.")
|
|
532
|
+
raw_paths.extend(str(item) for item in values or [])
|
|
533
|
+
return [safe_relative(project, item, label=f"{engine} environment dependency", must_exist=True) for item in raw_paths]
|
|
534
|
+
|
|
535
|
+
|
|
536
|
+
def fingerprint(
|
|
537
|
+
project: Path,
|
|
538
|
+
config: dict[str, Any],
|
|
539
|
+
config_path: Path,
|
|
540
|
+
record: dict[str, Any],
|
|
541
|
+
saved: dict[str, Any] | None = None,
|
|
542
|
+
) -> tuple[str, list[dict[str, Any]], dict[str, str]]:
|
|
543
|
+
image = resolve_image(config, record["engine"])
|
|
544
|
+
environment_identity = os.environ.get("UNALTRAWEB_COMPUTE_IMAGE_DIGEST", "").strip() or os.environ.get("UNALTRAWEB_COMPUTE_IMAGE_ID", "").strip()
|
|
545
|
+
local_identity = inspect_image(image["image"])
|
|
546
|
+
saved_image = saved.get("image") if isinstance(saved, dict) and isinstance(saved.get("image"), dict) else {}
|
|
547
|
+
saved_identity = saved.get("image_identity") if isinstance(saved, dict) and isinstance(saved.get("image_identity"), dict) else {}
|
|
548
|
+
recorded_identity = ""
|
|
549
|
+
if saved_image.get("image") == image["image"]:
|
|
550
|
+
recorded_identity = str(saved_identity.get("digest") or saved_identity.get("id") or "")
|
|
551
|
+
image_identity = environment_identity or local_identity.get("digest") or local_identity.get("id") or recorded_identity
|
|
552
|
+
dependencies = [record["source"], *record["inputs"], *engine_dependencies(project, config, record["engine"])]
|
|
553
|
+
if config_path.is_file():
|
|
554
|
+
dependencies.append(config_path)
|
|
555
|
+
entries: list[dict[str, Any]] = []
|
|
556
|
+
seen: set[str] = set()
|
|
557
|
+
for path in dependencies:
|
|
558
|
+
for entry in path_entries(project, path):
|
|
559
|
+
if entry["path"] not in seen:
|
|
560
|
+
entries.append(entry)
|
|
561
|
+
seen.add(entry["path"])
|
|
562
|
+
payload = {
|
|
563
|
+
"schema": 1,
|
|
564
|
+
"engine": record["engine"],
|
|
565
|
+
"image": image["image"],
|
|
566
|
+
"image_identity": image_identity,
|
|
567
|
+
"dependencies": entries,
|
|
568
|
+
}
|
|
569
|
+
if record.get("mode") == "figure":
|
|
570
|
+
payload["outputs"] = record["output_paths"]
|
|
571
|
+
else:
|
|
572
|
+
payload["output"] = record["output_path"]
|
|
573
|
+
payload["figures"] = record["figures_path"]
|
|
574
|
+
digest = hashlib.sha256(json.dumps(payload, sort_keys=True, separators=(",", ":")).encode("utf-8")).hexdigest()
|
|
575
|
+
return digest, entries, image
|
|
576
|
+
|
|
577
|
+
|
|
578
|
+
def file_signature(path: Path) -> dict[str, Any]:
|
|
579
|
+
return {"size": path.stat().st_size, "sha256": hashlib.sha256(path.read_bytes()).hexdigest()}
|
|
580
|
+
|
|
581
|
+
|
|
582
|
+
def tree_signatures(project: Path, root: Path) -> list[dict[str, Any]]:
|
|
583
|
+
if not root.is_dir():
|
|
584
|
+
return []
|
|
585
|
+
return [
|
|
586
|
+
{"path": relative(project, path), **file_signature(path)}
|
|
587
|
+
for path in sorted(item for item in root.rglob("*") if item.is_file())
|
|
588
|
+
]
|
|
589
|
+
|
|
590
|
+
|
|
591
|
+
def signatures_match(project: Path, entries: list[dict[str, Any]]) -> bool:
|
|
592
|
+
for entry in entries:
|
|
593
|
+
try:
|
|
594
|
+
path = safe_relative(project, str(entry["path"]), label="recorded artifact", must_exist=True)
|
|
595
|
+
except ComputationError:
|
|
596
|
+
return False
|
|
597
|
+
if file_signature(path) != {"size": entry.get("size"), "sha256": entry.get("sha256")}:
|
|
598
|
+
return False
|
|
599
|
+
return True
|
|
600
|
+
|
|
601
|
+
|
|
602
|
+
def no_unexpected_assets(project: Path, root: Path, entries: list[dict[str, Any]]) -> bool:
|
|
603
|
+
expected = {str(entry.get("path") or "") for entry in entries}
|
|
604
|
+
actual = {relative(project, path) for path in root.rglob("*") if path.is_file()} if root.is_dir() else set()
|
|
605
|
+
return actual == expected
|
|
606
|
+
|
|
607
|
+
|
|
608
|
+
def load_lock(project: Path) -> dict[str, Any]:
|
|
609
|
+
path = project / LOCK_PATH
|
|
610
|
+
if not path.is_file():
|
|
611
|
+
return {"version": 1, "records": {}}
|
|
612
|
+
try:
|
|
613
|
+
data = json.loads(path.read_text(encoding="utf-8"))
|
|
614
|
+
except json.JSONDecodeError as exc:
|
|
615
|
+
raise ComputationError(f"Invalid computation lock file: {exc}") from exc
|
|
616
|
+
if not isinstance(data, dict) or data.get("version") != 1 or not isinstance(data.get("records"), dict):
|
|
617
|
+
raise ComputationError("Invalid computations.lock.json structure.")
|
|
618
|
+
return data
|
|
619
|
+
|
|
620
|
+
|
|
621
|
+
def write_lock(project: Path, lock: dict[str, Any]) -> None:
|
|
622
|
+
path = project / LOCK_PATH
|
|
623
|
+
path.parent.mkdir(parents=True, exist_ok=True)
|
|
624
|
+
lock["version"] = 1
|
|
625
|
+
descriptor, raw_path = tempfile.mkstemp(prefix=f".{path.name}.", suffix=".tmp", dir=path.parent)
|
|
626
|
+
temporary = Path(raw_path)
|
|
627
|
+
try:
|
|
628
|
+
with os.fdopen(descriptor, "w", encoding="utf-8") as stream:
|
|
629
|
+
stream.write(json_dump(lock))
|
|
630
|
+
os.replace(temporary, path)
|
|
631
|
+
finally:
|
|
632
|
+
temporary.unlink(missing_ok=True)
|
|
633
|
+
|
|
634
|
+
|
|
635
|
+
def inspect_image(image: str) -> dict[str, str]:
|
|
636
|
+
if not shutil.which("docker"):
|
|
637
|
+
return {"available": "false", "id": "", "digest": ""}
|
|
638
|
+
command = ["docker", "image", "inspect", image, "--format", "{{.Id}}|{{join .RepoDigests \",\"}}"]
|
|
639
|
+
completed = subprocess.run(command, text=True, stdout=subprocess.PIPE, stderr=subprocess.DEVNULL, check=False)
|
|
640
|
+
if completed.returncode != 0:
|
|
641
|
+
return {"available": "false", "id": "", "digest": ""}
|
|
642
|
+
image_id, _, digest = completed.stdout.strip().partition("|")
|
|
643
|
+
return {"available": "true", "id": image_id, "digest": digest}
|
|
644
|
+
|
|
645
|
+
|
|
646
|
+
def status_item(project: Path, config: dict[str, Any], config_path: Path, record: dict[str, Any], saved: dict[str, Any]) -> dict[str, Any]:
|
|
647
|
+
current_fingerprint, dependencies, image = fingerprint(project, config, config_path, record, saved)
|
|
648
|
+
if record.get("mode") == "figure":
|
|
649
|
+
saved_outputs = saved.get("outputs") if isinstance(saved.get("outputs"), list) else []
|
|
650
|
+
outputs_exist = all(output.is_file() for output in record["outputs"])
|
|
651
|
+
outputs_valid = bool(outputs_exist and saved_outputs and signatures_match(project, saved_outputs))
|
|
652
|
+
current = bool(outputs_exist and not saved) or bool(saved and saved.get("fingerprint") == current_fingerprint and outputs_valid)
|
|
653
|
+
reason = "current"
|
|
654
|
+
if not saved and outputs_exist:
|
|
655
|
+
reason = "present_unmanaged"
|
|
656
|
+
elif not saved:
|
|
657
|
+
reason = "not_rendered"
|
|
658
|
+
elif saved.get("fingerprint") != current_fingerprint:
|
|
659
|
+
reason = "source_or_environment_changed"
|
|
660
|
+
elif not outputs_exist:
|
|
661
|
+
reason = "output_missing"
|
|
662
|
+
elif not outputs_valid:
|
|
663
|
+
reason = "output_modified"
|
|
664
|
+
return {
|
|
665
|
+
"source": record["source_path"],
|
|
666
|
+
"mode": "figure",
|
|
667
|
+
"outputs": record["output_paths"],
|
|
668
|
+
"engine": record["engine"],
|
|
669
|
+
"image": image,
|
|
670
|
+
"local_image": inspect_image(image["image"]),
|
|
671
|
+
"fingerprint": current_fingerprint,
|
|
672
|
+
"dependencies": dependencies,
|
|
673
|
+
"current": current,
|
|
674
|
+
"reason": reason,
|
|
675
|
+
}
|
|
676
|
+
output_exists = record["output"].is_file()
|
|
677
|
+
output_valid = bool(output_exists and saved.get("output") and signatures_match(project, [saved["output"]]))
|
|
678
|
+
saved_assets = saved.get("assets", [])
|
|
679
|
+
assets_valid = signatures_match(project, saved_assets) and no_unexpected_assets(project, record["figures"], saved_assets)
|
|
680
|
+
current = bool(output_exists and not saved) or bool(saved and saved.get("fingerprint") == current_fingerprint and output_valid and assets_valid)
|
|
681
|
+
reason = "current"
|
|
682
|
+
if not saved and output_exists:
|
|
683
|
+
reason = "present_unmanaged"
|
|
684
|
+
elif not saved:
|
|
685
|
+
reason = "not_rendered"
|
|
686
|
+
elif saved.get("fingerprint") != current_fingerprint:
|
|
687
|
+
reason = "source_or_environment_changed"
|
|
688
|
+
elif not output_exists:
|
|
689
|
+
reason = "output_missing"
|
|
690
|
+
elif not output_valid:
|
|
691
|
+
reason = "output_modified"
|
|
692
|
+
elif not assets_valid:
|
|
693
|
+
reason = "assets_missing_or_modified"
|
|
694
|
+
return {
|
|
695
|
+
"source": record["source_path"],
|
|
696
|
+
"mode": "chapter",
|
|
697
|
+
"output": record["output_path"],
|
|
698
|
+
"figures": record["figures_path"],
|
|
699
|
+
"engine": record["engine"],
|
|
700
|
+
"image": image,
|
|
701
|
+
"local_image": inspect_image(image["image"]),
|
|
702
|
+
"fingerprint": current_fingerprint,
|
|
703
|
+
"dependencies": dependencies,
|
|
704
|
+
"current": current,
|
|
705
|
+
"reason": reason,
|
|
706
|
+
}
|
|
707
|
+
|
|
708
|
+
|
|
709
|
+
def status(project: Path, source: str = "", engine: str = "", mode: str = "") -> dict[str, Any]:
|
|
710
|
+
config, config_path = load_config(project)
|
|
711
|
+
all_records = discover_sources(project, config)
|
|
712
|
+
records = [
|
|
713
|
+
item for item in all_records
|
|
714
|
+
if (not engine or item["engine"] == engine) and (not mode or item["mode"] == mode)
|
|
715
|
+
]
|
|
716
|
+
if source:
|
|
717
|
+
selected = safe_relative(project, source, label="selected computation source", must_exist=True)
|
|
718
|
+
records = [item for item in records if item["source"] == selected]
|
|
719
|
+
if not records:
|
|
720
|
+
raise ComputationError(f"Selected file is not an enabled computation source: {source}")
|
|
721
|
+
lock = load_lock(project)
|
|
722
|
+
lock_records = lock["records"]
|
|
723
|
+
items = [status_item(project, config, config_path, record, lock_records.get(record["source_path"], {})) for record in records]
|
|
724
|
+
known = {item["source_path"] for item in all_records}
|
|
725
|
+
orphaned = sorted(path for path in lock_records if path not in known)
|
|
726
|
+
return {
|
|
727
|
+
"project": str(project),
|
|
728
|
+
"enabled": config["enabled"],
|
|
729
|
+
"config": str(CONFIG_PATH) if config_path.is_file() else "",
|
|
730
|
+
"lock": str(LOCK_PATH) if (project / LOCK_PATH).is_file() else "",
|
|
731
|
+
"sources": items,
|
|
732
|
+
"source_count": len(items),
|
|
733
|
+
"current_count": sum(1 for item in items if item["current"]),
|
|
734
|
+
"orphaned_records": orphaned,
|
|
735
|
+
"ok": all(item["current"] for item in items) and not orphaned,
|
|
736
|
+
}
|
|
737
|
+
|
|
738
|
+
|
|
739
|
+
def run_command(command: list[str]) -> None:
|
|
740
|
+
completed = subprocess.run(command, check=False)
|
|
741
|
+
if completed.returncode != 0:
|
|
742
|
+
raise ComputationError(f"Command failed with exit code {completed.returncode}: {' '.join(command)}")
|
|
743
|
+
|
|
744
|
+
|
|
745
|
+
def locate_rendered_markdown(stage: Path, source: Path) -> Path:
|
|
746
|
+
candidates = sorted(stage.rglob("*.md"))
|
|
747
|
+
preferred = [path for path in candidates if path.stem == source.stem]
|
|
748
|
+
if len(preferred) == 1:
|
|
749
|
+
return preferred[0]
|
|
750
|
+
if len(candidates) == 1:
|
|
751
|
+
return candidates[0]
|
|
752
|
+
names = ", ".join(str(path.relative_to(stage)) for path in candidates)
|
|
753
|
+
raise ComputationError(f"Expected one rendered Markdown output for {source.name}; found: {names or 'none'}")
|
|
754
|
+
|
|
755
|
+
|
|
756
|
+
def split_generated_markdown(text: str) -> str:
|
|
757
|
+
match = FRONT_MATTER_RE.match(text)
|
|
758
|
+
return text[match.end():] if match else text
|
|
759
|
+
|
|
760
|
+
|
|
761
|
+
def normalize_generated_body(text: str, title: str) -> str:
|
|
762
|
+
lines = text.strip().splitlines()
|
|
763
|
+
if lines and lines[0].strip() == f"# {title}":
|
|
764
|
+
lines.pop(0)
|
|
765
|
+
while lines and not lines[0].strip():
|
|
766
|
+
lines.pop(0)
|
|
767
|
+
elif len(lines) >= 2 and lines[0].strip() == title and re.fullmatch(r"=+", lines[1].strip()):
|
|
768
|
+
lines = lines[2:]
|
|
769
|
+
while lines and not lines[0].strip():
|
|
770
|
+
lines.pop(0)
|
|
771
|
+
return "\n".join(lines).strip()
|
|
772
|
+
|
|
773
|
+
|
|
774
|
+
def yaml_front(front: dict[str, Any]) -> str:
|
|
775
|
+
public = {key: value for key, value in front.items() if key != "unaltraweb_compute"}
|
|
776
|
+
dumped = yaml.safe_dump(public, allow_unicode=True, sort_keys=False).rstrip() if yaml is not None else simple_yaml_dump(public).rstrip()
|
|
777
|
+
return "---\n" + dumped + "\n---\n"
|
|
778
|
+
|
|
779
|
+
|
|
780
|
+
def generated_media(stage: Path, markdown: Path, text: str, figures_path: str) -> tuple[str, list[tuple[Path, Path]]]:
|
|
781
|
+
copies: dict[str, tuple[Path, Path]] = {}
|
|
782
|
+
|
|
783
|
+
def quarto_figure(match: re.Match[str]) -> str:
|
|
784
|
+
attributes, raw_caption = match.groups()
|
|
785
|
+
|
|
786
|
+
def attribute(name: str) -> str:
|
|
787
|
+
found = re.search(rf'\b{re.escape(name)}="([^"]*)"', attributes, re.IGNORECASE)
|
|
788
|
+
return html.unescape(found.group(1)) if found else ""
|
|
789
|
+
|
|
790
|
+
raw_path = attribute("src")
|
|
791
|
+
alt = attribute("data-fig-alt") or attribute("alt")
|
|
792
|
+
caption = html.unescape(re.sub(r"<[^>]+>", "", raw_caption)).replace("\u00a0", " ").strip()
|
|
793
|
+
caption = re.sub(r"^Figure\s+\d+\s*:\s*", "", caption, flags=re.IGNORECASE)
|
|
794
|
+
escaped = caption.replace('"', """)
|
|
795
|
+
return f''
|
|
796
|
+
|
|
797
|
+
text = QUARTO_FIGURE_RE.sub(quarto_figure, text)
|
|
798
|
+
|
|
799
|
+
def image(match: re.Match[str]) -> str:
|
|
800
|
+
alt, raw_path, title, attributes = match.groups()
|
|
801
|
+
if re.match(r"^[a-z][a-z0-9+.-]*://", raw_path, re.IGNORECASE) or raw_path.startswith("{{"):
|
|
802
|
+
return match.group(0)
|
|
803
|
+
if raw_path.startswith("/output/"):
|
|
804
|
+
candidate = stage / raw_path.removeprefix("/output/")
|
|
805
|
+
else:
|
|
806
|
+
candidate = (markdown.parent / raw_path).resolve()
|
|
807
|
+
try:
|
|
808
|
+
candidate.relative_to(stage)
|
|
809
|
+
except ValueError:
|
|
810
|
+
return match.group(0)
|
|
811
|
+
if not candidate.is_file():
|
|
812
|
+
return match.group(0)
|
|
813
|
+
if candidate.suffix.lower() not in GENERATED_MEDIA_SUFFIXES:
|
|
814
|
+
raise ComputationError(f"Unsupported generated media type: {candidate.name}")
|
|
815
|
+
relative_media = candidate.relative_to(markdown.parent)
|
|
816
|
+
if ".." in relative_media.parts:
|
|
817
|
+
relative_media = Path(candidate.name)
|
|
818
|
+
target_relative = Path(figures_path) / relative_media
|
|
819
|
+
copies[str(target_relative)] = (candidate, target_relative)
|
|
820
|
+
caption = (title or alt).strip()
|
|
821
|
+
if not caption:
|
|
822
|
+
raise ComputationError(f"Generated figure requires fig-cap or fig-alt: {raw_path}")
|
|
823
|
+
escaped = caption.replace('"', """)
|
|
824
|
+
return f'.replace(os.sep, "/")} "{escaped}"){attributes or ""}'
|
|
825
|
+
|
|
826
|
+
rendered = IMAGE_RE.sub(image, text)
|
|
827
|
+
return rendered, list(copies.values())
|
|
828
|
+
|
|
829
|
+
|
|
830
|
+
def render_stage(project: Path, config: dict[str, Any], record: dict[str, Any], stage: Path, image: str) -> tuple[str, list[tuple[Path, Path]], dict[str, str]]:
|
|
831
|
+
command = [
|
|
832
|
+
"quarto",
|
|
833
|
+
"render",
|
|
834
|
+
record["source_path"],
|
|
835
|
+
"--to",
|
|
836
|
+
"gfm+yaml_metadata_block",
|
|
837
|
+
"--output-dir",
|
|
838
|
+
str(stage),
|
|
839
|
+
"--execute",
|
|
840
|
+
"--no-cache",
|
|
841
|
+
"--no-clean",
|
|
842
|
+
]
|
|
843
|
+
run_command(command)
|
|
844
|
+
image_identity = {
|
|
845
|
+
"available": "true",
|
|
846
|
+
"id": os.environ.get("UNALTRAWEB_COMPUTE_IMAGE_ID", ""),
|
|
847
|
+
"digest": os.environ.get("UNALTRAWEB_COMPUTE_IMAGE_DIGEST", ""),
|
|
848
|
+
}
|
|
849
|
+
markdown = locate_rendered_markdown(stage, record["source"])
|
|
850
|
+
body = normalize_generated_body(
|
|
851
|
+
split_generated_markdown(markdown.read_text(encoding="utf-8")),
|
|
852
|
+
str(record["front"]["title"]),
|
|
853
|
+
)
|
|
854
|
+
body, media = generated_media(stage, markdown, body, record["figures_path"])
|
|
855
|
+
marker = f'<!-- Generated from {record["source_path"]} by unaltraweb computations. Do not edit this file directly. -->'
|
|
856
|
+
output_text = yaml_front(record["front"]) + "\n" + marker + "\n\n" + body + "\n"
|
|
857
|
+
return output_text, media, image_identity
|
|
858
|
+
|
|
859
|
+
|
|
860
|
+
def staged_figure_output(stage: Path, output_path: str) -> Path | None:
|
|
861
|
+
direct = stage / output_path
|
|
862
|
+
if direct.is_file():
|
|
863
|
+
return direct
|
|
864
|
+
matches = sorted(path for path in stage.rglob(Path(output_path).name) if path.is_file())
|
|
865
|
+
return matches[0] if len(matches) == 1 else None
|
|
866
|
+
|
|
867
|
+
|
|
868
|
+
def publish_figure_outputs(record: dict[str, Any], stage: Path, confirm_overwrite: bool, owned: bool) -> None:
|
|
869
|
+
staged: list[tuple[Path, Path]] = []
|
|
870
|
+
missing: list[Path] = []
|
|
871
|
+
for output, output_path in zip(record["outputs"], record["output_paths"]):
|
|
872
|
+
candidate = staged_figure_output(stage, output_path)
|
|
873
|
+
if candidate:
|
|
874
|
+
staged.append((candidate, output))
|
|
875
|
+
elif not output.is_file():
|
|
876
|
+
missing.append(output)
|
|
877
|
+
if missing:
|
|
878
|
+
names = ", ".join(str(path) for path in missing)
|
|
879
|
+
raise ComputationError(f"Figure computation did not create declared output(s): {names}")
|
|
880
|
+
if not staged:
|
|
881
|
+
return
|
|
882
|
+
unmanaged = [output for _, output in staged if output.exists()]
|
|
883
|
+
if unmanaged and not owned and not confirm_overwrite:
|
|
884
|
+
names = ", ".join(str(path) for path in unmanaged)
|
|
885
|
+
raise ComputationError(
|
|
886
|
+
f"Refusing to replace unmanaged generated figure output(s) for {record['source_path']}: {names}; rerun with --confirm-overwrite after review."
|
|
887
|
+
)
|
|
888
|
+
token = secrets.token_hex(6)
|
|
889
|
+
replacements: list[tuple[Path, Path, Path | None]] = []
|
|
890
|
+
try:
|
|
891
|
+
for source, output in staged:
|
|
892
|
+
output.parent.mkdir(parents=True, exist_ok=True)
|
|
893
|
+
temp_output = output.with_name(f".{output.name}.{token}.compute-tmp")
|
|
894
|
+
backup_output = output.with_name(f".{output.name}.{token}.compute-backup")
|
|
895
|
+
if temp_output.exists():
|
|
896
|
+
temp_output.unlink()
|
|
897
|
+
if backup_output.exists():
|
|
898
|
+
backup_output.unlink()
|
|
899
|
+
shutil.copy2(source, temp_output)
|
|
900
|
+
if output.exists():
|
|
901
|
+
os.replace(output, backup_output)
|
|
902
|
+
replacements.append((temp_output, output, backup_output))
|
|
903
|
+
else:
|
|
904
|
+
replacements.append((temp_output, output, None))
|
|
905
|
+
os.replace(temp_output, output)
|
|
906
|
+
except Exception:
|
|
907
|
+
for temp_output, output, backup_output in replacements:
|
|
908
|
+
if output.exists():
|
|
909
|
+
output.unlink()
|
|
910
|
+
if backup_output and backup_output.exists():
|
|
911
|
+
os.replace(backup_output, output)
|
|
912
|
+
if temp_output.exists():
|
|
913
|
+
temp_output.unlink()
|
|
914
|
+
raise
|
|
915
|
+
finally:
|
|
916
|
+
for _, _, backup_output in replacements:
|
|
917
|
+
if backup_output and backup_output.exists():
|
|
918
|
+
backup_output.unlink()
|
|
919
|
+
|
|
920
|
+
|
|
921
|
+
def render_figure_outputs(record: dict[str, Any], stage: Path, confirm_overwrite: bool, owned: bool) -> dict[str, str]:
|
|
922
|
+
command = [
|
|
923
|
+
"quarto",
|
|
924
|
+
"render",
|
|
925
|
+
record["source_path"],
|
|
926
|
+
"--to",
|
|
927
|
+
"gfm+yaml_metadata_block",
|
|
928
|
+
"--output-dir",
|
|
929
|
+
str(stage),
|
|
930
|
+
"--execute",
|
|
931
|
+
"--no-cache",
|
|
932
|
+
"--no-clean",
|
|
933
|
+
]
|
|
934
|
+
run_command(command)
|
|
935
|
+
publish_figure_outputs(record, stage, confirm_overwrite, owned)
|
|
936
|
+
return {
|
|
937
|
+
"available": "true",
|
|
938
|
+
"id": os.environ.get("UNALTRAWEB_COMPUTE_IMAGE_ID", ""),
|
|
939
|
+
"digest": os.environ.get("UNALTRAWEB_COMPUTE_IMAGE_DIGEST", ""),
|
|
940
|
+
}
|
|
941
|
+
|
|
942
|
+
|
|
943
|
+
def saved_paths_match(record: dict[str, Any], saved: dict[str, Any]) -> bool:
|
|
944
|
+
output = saved.get("output") if isinstance(saved.get("output"), dict) else {}
|
|
945
|
+
return output.get("path") == record["output_path"] and saved.get("figures") == record["figures_path"]
|
|
946
|
+
|
|
947
|
+
|
|
948
|
+
def saved_figure_outputs_match(record: dict[str, Any], saved: dict[str, Any]) -> bool:
|
|
949
|
+
outputs = saved.get("outputs") if isinstance(saved.get("outputs"), list) else []
|
|
950
|
+
return [str(item.get("path") or "") for item in outputs] == record["output_paths"]
|
|
951
|
+
|
|
952
|
+
|
|
953
|
+
def validate_path_migration(
|
|
954
|
+
project: Path,
|
|
955
|
+
config: dict[str, Any],
|
|
956
|
+
record: dict[str, Any],
|
|
957
|
+
saved: dict[str, Any],
|
|
958
|
+
confirm_overwrite: bool,
|
|
959
|
+
other_records: list[dict[str, Any]],
|
|
960
|
+
) -> tuple[Path | None, Path | None]:
|
|
961
|
+
if not saved or saved_paths_match(record, saved):
|
|
962
|
+
return None, None
|
|
963
|
+
if not confirm_overwrite:
|
|
964
|
+
raise ComputationError(f"Changing managed output paths for {record['source_path']} requires --confirm-overwrite after review.")
|
|
965
|
+
saved_output_data = saved.get("output") if isinstance(saved.get("output"), dict) else {}
|
|
966
|
+
saved_output_raw = str(saved_output_data.get("path") or "")
|
|
967
|
+
saved_figures_raw = str(saved.get("figures") or "")
|
|
968
|
+
old_output = safe_relative(project, saved_output_raw, label="previous generated Markdown output") if saved_output_raw else None
|
|
969
|
+
old_figures = safe_relative(project, saved_figures_raw, label="previous generated figures directory") if saved_figures_raw else None
|
|
970
|
+
roots = source_roots(project, config)
|
|
971
|
+
assets_root = safe_relative(project, config["generated_assets_root"], label="generated assets root")
|
|
972
|
+
if old_output and (old_output.suffix.lower() != ".md" or not any(contains(root, old_output) for root in roots)):
|
|
973
|
+
raise ComputationError(f"Recorded previous Markdown path is outside managed source roots: {saved_output_raw}")
|
|
974
|
+
if old_figures and (old_figures == assets_root or not contains(assets_root, old_figures)):
|
|
975
|
+
raise ComputationError(f"Recorded previous figures path is outside the managed assets root: {saved_figures_raw}")
|
|
976
|
+
protected: list[Path] = []
|
|
977
|
+
for item in other_records:
|
|
978
|
+
if item.get("mode") == "figure":
|
|
979
|
+
protected.extend(item["outputs"])
|
|
980
|
+
else:
|
|
981
|
+
protected.extend([item["output"], item["figures"]])
|
|
982
|
+
if any(old_path and any(paths_overlap(old_path, path) for path in protected) for old_path in [old_output, old_figures]):
|
|
983
|
+
raise ComputationError(f"Recorded previous paths overlap another source's current output: {record['source_path']}")
|
|
984
|
+
for old_path in [old_output, old_figures]:
|
|
985
|
+
if old_path and any(paths_overlap(old_path, current) and old_path != current for current in [record["output"], record["figures"]]):
|
|
986
|
+
raise ComputationError(f"Previous and new generated paths overlap for {record['source_path']}; move them manually after review.")
|
|
987
|
+
if old_output and old_output.exists() and not signatures_match(project, [saved_output_data]):
|
|
988
|
+
raise ComputationError(f"Previous generated Markdown was modified and will not be removed: {saved_output_raw}")
|
|
989
|
+
saved_assets = saved.get("assets") if isinstance(saved.get("assets"), list) else []
|
|
990
|
+
if old_figures and old_figures.exists() and (not signatures_match(project, saved_assets) or not no_unexpected_assets(project, old_figures, saved_assets)):
|
|
991
|
+
raise ComputationError(f"Previous generated figures were modified and will not be removed: {saved_figures_raw}")
|
|
992
|
+
return old_output, old_figures
|
|
993
|
+
|
|
994
|
+
|
|
995
|
+
def remove_previous_paths(record: dict[str, Any], old_output: Path | None, old_figures: Path | None) -> None:
|
|
996
|
+
if old_output and old_output != record["output"]:
|
|
997
|
+
old_output.unlink(missing_ok=True)
|
|
998
|
+
if old_figures and old_figures != record["figures"] and old_figures.is_dir():
|
|
999
|
+
shutil.rmtree(old_figures)
|
|
1000
|
+
|
|
1001
|
+
|
|
1002
|
+
def record_managed_paths(record: dict[str, Any]) -> list[Path]:
|
|
1003
|
+
if record.get("mode") == "figure":
|
|
1004
|
+
return list(record["outputs"])
|
|
1005
|
+
return [record["output"], record["figures"]]
|
|
1006
|
+
|
|
1007
|
+
|
|
1008
|
+
def orphan_paths(project: Path, saved: dict[str, Any]) -> list[Path]:
|
|
1009
|
+
if saved.get("mode") == "figure":
|
|
1010
|
+
outputs = saved.get("outputs") if isinstance(saved.get("outputs"), list) else []
|
|
1011
|
+
paths: list[Path] = []
|
|
1012
|
+
for entry in outputs:
|
|
1013
|
+
raw = str(entry.get("path") or "") if isinstance(entry, dict) else ""
|
|
1014
|
+
if raw:
|
|
1015
|
+
paths.append(safe_relative(project, raw, label="orphaned generated figure"))
|
|
1016
|
+
return paths
|
|
1017
|
+
paths = []
|
|
1018
|
+
output = saved.get("output") if isinstance(saved.get("output"), dict) else {}
|
|
1019
|
+
output_path = str(output.get("path") or "")
|
|
1020
|
+
figures_path = str(saved.get("figures") or "")
|
|
1021
|
+
if output_path:
|
|
1022
|
+
paths.append(safe_relative(project, output_path, label="orphaned generated Markdown"))
|
|
1023
|
+
if figures_path:
|
|
1024
|
+
paths.append(safe_relative(project, figures_path, label="orphaned generated figures"))
|
|
1025
|
+
return paths
|
|
1026
|
+
|
|
1027
|
+
|
|
1028
|
+
def prune_absent_orphan_records(project: Path, lock: dict[str, Any], records: list[dict[str, Any]]) -> None:
|
|
1029
|
+
known = {item["source_path"] for item in records}
|
|
1030
|
+
managed_paths = [path for record in records for path in record_managed_paths(record)]
|
|
1031
|
+
for source_path in [path for path in lock["records"] if path not in known]:
|
|
1032
|
+
saved = lock["records"][source_path]
|
|
1033
|
+
try:
|
|
1034
|
+
saved_paths = orphan_paths(project, saved)
|
|
1035
|
+
except ComputationError:
|
|
1036
|
+
saved_paths = []
|
|
1037
|
+
if saved_paths and all(any(paths_overlap(saved_path, managed_path) for managed_path in managed_paths) for saved_path in saved_paths):
|
|
1038
|
+
del lock["records"][source_path]
|
|
1039
|
+
continue
|
|
1040
|
+
if saved.get("mode") == "figure":
|
|
1041
|
+
if not any(path.exists() for path in saved_paths):
|
|
1042
|
+
del lock["records"][source_path]
|
|
1043
|
+
continue
|
|
1044
|
+
if not any(path.exists() for path in saved_paths):
|
|
1045
|
+
del lock["records"][source_path]
|
|
1046
|
+
|
|
1047
|
+
|
|
1048
|
+
def publish_render(project: Path, record: dict[str, Any], output_text: str, media: list[tuple[Path, Path]], confirm_overwrite: bool, owned: bool) -> None:
|
|
1049
|
+
output = record["output"]
|
|
1050
|
+
figures = record["figures"]
|
|
1051
|
+
if not owned and (output.exists() or figures.exists()) and not confirm_overwrite:
|
|
1052
|
+
raise ComputationError(
|
|
1053
|
+
f"Refusing to replace unmanaged generated paths for {record['source_path']}; rerun with --confirm-overwrite after review."
|
|
1054
|
+
)
|
|
1055
|
+
output.parent.mkdir(parents=True, exist_ok=True)
|
|
1056
|
+
figures.parent.mkdir(parents=True, exist_ok=True)
|
|
1057
|
+
token = secrets.token_hex(6)
|
|
1058
|
+
temp_output = output.with_name(f".{output.name}.{token}.compute-tmp")
|
|
1059
|
+
temp_figures = figures.with_name(f".{figures.name}.{token}.compute-tmp")
|
|
1060
|
+
backup_output = output.with_name(f".{output.name}.{token}.compute-backup")
|
|
1061
|
+
backup_figures = figures.with_name(f".{figures.name}.{token}.compute-backup")
|
|
1062
|
+
for path in [temp_output, temp_figures, backup_output, backup_figures]:
|
|
1063
|
+
if path.is_dir():
|
|
1064
|
+
shutil.rmtree(path)
|
|
1065
|
+
elif path.exists():
|
|
1066
|
+
path.unlink()
|
|
1067
|
+
temp_output.write_text(output_text, encoding="utf-8")
|
|
1068
|
+
temp_figures.mkdir(parents=True)
|
|
1069
|
+
for source, target_relative in media:
|
|
1070
|
+
target = temp_figures / Path(target_relative).relative_to(Path(record["figures_path"]))
|
|
1071
|
+
target.parent.mkdir(parents=True, exist_ok=True)
|
|
1072
|
+
shutil.copy2(source, target)
|
|
1073
|
+
try:
|
|
1074
|
+
if output.exists():
|
|
1075
|
+
os.replace(output, backup_output)
|
|
1076
|
+
if figures.exists():
|
|
1077
|
+
os.replace(figures, backup_figures)
|
|
1078
|
+
os.replace(temp_figures, figures)
|
|
1079
|
+
os.replace(temp_output, output)
|
|
1080
|
+
except Exception:
|
|
1081
|
+
if output.exists():
|
|
1082
|
+
output.unlink()
|
|
1083
|
+
if figures.exists():
|
|
1084
|
+
shutil.rmtree(figures)
|
|
1085
|
+
if backup_output.exists():
|
|
1086
|
+
os.replace(backup_output, output)
|
|
1087
|
+
if backup_figures.exists():
|
|
1088
|
+
os.replace(backup_figures, figures)
|
|
1089
|
+
raise
|
|
1090
|
+
finally:
|
|
1091
|
+
if backup_output.exists():
|
|
1092
|
+
backup_output.unlink()
|
|
1093
|
+
if backup_figures.exists():
|
|
1094
|
+
shutil.rmtree(backup_figures)
|
|
1095
|
+
|
|
1096
|
+
|
|
1097
|
+
@contextlib.contextmanager
|
|
1098
|
+
def project_render_lock(project: Path):
|
|
1099
|
+
root = project / "tmp" / "manual-computations"
|
|
1100
|
+
root.mkdir(parents=True, exist_ok=True)
|
|
1101
|
+
with (root / ".render.lock").open("w", encoding="utf-8") as stream:
|
|
1102
|
+
fcntl.flock(stream.fileno(), fcntl.LOCK_EX)
|
|
1103
|
+
yield
|
|
1104
|
+
|
|
1105
|
+
|
|
1106
|
+
def _render(project: Path, source: str = "", engine: str = "", confirm_overwrite: bool = False, stale_only: bool = False, mode: str = "") -> dict[str, Any]:
|
|
1107
|
+
config, config_path = load_config(project)
|
|
1108
|
+
all_records = discover_sources(project, config)
|
|
1109
|
+
records = [
|
|
1110
|
+
item for item in all_records
|
|
1111
|
+
if (not engine or item["engine"] == engine) and (not mode or item["mode"] == mode)
|
|
1112
|
+
]
|
|
1113
|
+
if source:
|
|
1114
|
+
selected = safe_relative(project, source, label="selected computation source", must_exist=True)
|
|
1115
|
+
records = [item for item in records if item["source"] == selected]
|
|
1116
|
+
if not records:
|
|
1117
|
+
raise ComputationError(f"Selected file is not an enabled computation source: {source}")
|
|
1118
|
+
lock = load_lock(project)
|
|
1119
|
+
results: list[dict[str, Any]] = []
|
|
1120
|
+
for record in records:
|
|
1121
|
+
saved = lock["records"].get(record["source_path"], {})
|
|
1122
|
+
if stale_only:
|
|
1123
|
+
if status_item(project, config, config_path, record, saved)["current"]:
|
|
1124
|
+
continue
|
|
1125
|
+
managed_paths = list(record["outputs"]) if record.get("mode") == "figure" else [record["output"], record["figures"]]
|
|
1126
|
+
if not saved and all(path.exists() for path in managed_paths):
|
|
1127
|
+
continue
|
|
1128
|
+
current_fingerprint, dependencies, image = fingerprint(project, config, config_path, record, saved)
|
|
1129
|
+
saved = lock["records"].get(record["source_path"], {})
|
|
1130
|
+
if record.get("mode") == "figure":
|
|
1131
|
+
build_root = project / "tmp" / "manual-computations"
|
|
1132
|
+
build_root.mkdir(parents=True, exist_ok=True)
|
|
1133
|
+
with tempfile.TemporaryDirectory(prefix=f"{record['source'].stem}-", dir=build_root) as temporary:
|
|
1134
|
+
image_identity = render_figure_outputs(record, Path(temporary), confirm_overwrite, bool(saved and saved_figure_outputs_match(record, saved)))
|
|
1135
|
+
lock["records"][record["source_path"]] = {
|
|
1136
|
+
"mode": "figure",
|
|
1137
|
+
"engine": record["engine"],
|
|
1138
|
+
"image": image,
|
|
1139
|
+
"image_identity": image_identity,
|
|
1140
|
+
"fingerprint": current_fingerprint,
|
|
1141
|
+
"dependencies": dependencies,
|
|
1142
|
+
"outputs": [{"path": relative(project, output), **file_signature(output)} for output in record["outputs"]],
|
|
1143
|
+
"rendered_at": utc_now(),
|
|
1144
|
+
}
|
|
1145
|
+
write_lock(project, lock)
|
|
1146
|
+
results.append({"source": record["source_path"], "mode": "figure", "outputs": record["output_paths"], "engine": record["engine"], "image": image})
|
|
1147
|
+
continue
|
|
1148
|
+
other_records = [item for item in all_records if item["source_path"] != record["source_path"]]
|
|
1149
|
+
old_output, old_figures = validate_path_migration(project, config, record, saved, confirm_overwrite, other_records)
|
|
1150
|
+
build_root = project / "tmp" / "manual-computations"
|
|
1151
|
+
build_root.mkdir(parents=True, exist_ok=True)
|
|
1152
|
+
with tempfile.TemporaryDirectory(prefix=f"{record['source'].stem}-", dir=build_root) as temporary:
|
|
1153
|
+
output_text, media, image_identity = render_stage(project, config, record, Path(temporary), image["image"])
|
|
1154
|
+
publish_render(project, record, output_text, media, confirm_overwrite, bool(saved and saved_paths_match(record, saved)))
|
|
1155
|
+
remove_previous_paths(record, old_output, old_figures)
|
|
1156
|
+
lock["records"][record["source_path"]] = {
|
|
1157
|
+
"engine": record["engine"],
|
|
1158
|
+
"image": image,
|
|
1159
|
+
"image_identity": image_identity,
|
|
1160
|
+
"fingerprint": current_fingerprint,
|
|
1161
|
+
"dependencies": dependencies,
|
|
1162
|
+
"output": {"path": record["output_path"], **file_signature(record["output"])},
|
|
1163
|
+
"figures": record["figures_path"],
|
|
1164
|
+
"assets": tree_signatures(project, record["figures"]),
|
|
1165
|
+
"rendered_at": utc_now(),
|
|
1166
|
+
}
|
|
1167
|
+
write_lock(project, lock)
|
|
1168
|
+
results.append({"source": record["source_path"], "output": record["output_path"], "figures": record["figures_path"], "engine": record["engine"], "image": image, "assets": len(media)})
|
|
1169
|
+
if not source and not engine:
|
|
1170
|
+
prune_absent_orphan_records(project, lock, all_records)
|
|
1171
|
+
write_lock(project, lock)
|
|
1172
|
+
return {"project": str(project), "rendered": results, "rendered_count": len(results), "ok": True}
|
|
1173
|
+
|
|
1174
|
+
|
|
1175
|
+
def render(project: Path, source: str = "", engine: str = "", confirm_overwrite: bool = False, stale_only: bool = False, mode: str = "") -> dict[str, Any]:
|
|
1176
|
+
with project_render_lock(project):
|
|
1177
|
+
return _render(project, source=source, engine=engine, mode=mode, confirm_overwrite=confirm_overwrite, stale_only=stale_only)
|
|
1178
|
+
|
|
1179
|
+
|
|
1180
|
+
def prune(project: Path) -> dict[str, Any]:
|
|
1181
|
+
with project_render_lock(project):
|
|
1182
|
+
config, _ = load_config(project)
|
|
1183
|
+
records = discover_sources(project, config)
|
|
1184
|
+
lock = load_lock(project)
|
|
1185
|
+
before = set(lock["records"])
|
|
1186
|
+
prune_absent_orphan_records(project, lock, records)
|
|
1187
|
+
removed = sorted(before - set(lock["records"]))
|
|
1188
|
+
write_lock(project, lock)
|
|
1189
|
+
return {"project": str(project), "removed_records": removed, "removed_count": len(removed), "ok": True}
|
|
1190
|
+
|
|
1191
|
+
|
|
1192
|
+
def build_engine_image(project: Path, config: dict[str, Any], engine: str) -> dict[str, Any]:
|
|
1193
|
+
settings = engine_settings(config, engine)
|
|
1194
|
+
selected = resolve_image(config, engine)
|
|
1195
|
+
dockerfile_raw = str(settings.get("dockerfile") or "").strip()
|
|
1196
|
+
local_image = str(settings.get("local_image") or default_local_image(config, engine)).strip()
|
|
1197
|
+
if not dockerfile_raw or compute_environment() != "local" or selected["image"] != local_image:
|
|
1198
|
+
image = selected["image"]
|
|
1199
|
+
if inspect_image(image)["available"] == "true":
|
|
1200
|
+
return {"engine": engine, "action": "reuse", "image": image, "ok": True}
|
|
1201
|
+
run_command(["docker", "pull", image])
|
|
1202
|
+
return {"engine": engine, "action": "pull", "image": image, "ok": True}
|
|
1203
|
+
dockerfile = safe_relative(project, dockerfile_raw, label=f"{engine} Dockerfile", must_exist=True)
|
|
1204
|
+
context = safe_relative(project, str(settings.get("context") or "."), label=f"{engine} Docker context", must_exist=True)
|
|
1205
|
+
base_image = str(settings.get("base_image") or DEFAULT_IMAGES[engine]).strip()
|
|
1206
|
+
command = ["docker", "build"]
|
|
1207
|
+
network = os.environ.get("COMPUTE_DOCKER_BUILD_NETWORK", "").strip()
|
|
1208
|
+
if network:
|
|
1209
|
+
command.extend(["--network", network])
|
|
1210
|
+
command.extend(
|
|
1211
|
+
[
|
|
1212
|
+
"--build-arg",
|
|
1213
|
+
f"BASE_IMAGE={base_image}",
|
|
1214
|
+
"-f",
|
|
1215
|
+
str(dockerfile),
|
|
1216
|
+
"-t",
|
|
1217
|
+
local_image,
|
|
1218
|
+
str(context),
|
|
1219
|
+
]
|
|
1220
|
+
)
|
|
1221
|
+
run_command(command)
|
|
1222
|
+
return {"engine": engine, "action": "build", "image": local_image, "base_image": base_image, "dockerfile": relative(project, dockerfile), "context": relative(project, context), "ok": True}
|
|
1223
|
+
|
|
1224
|
+
|
|
1225
|
+
def parser() -> argparse.ArgumentParser:
|
|
1226
|
+
root = argparse.ArgumentParser(prog="unaltraweb-computations")
|
|
1227
|
+
root.add_argument("command", choices=["status", "check", "render", "prune", "resolve", "image"])
|
|
1228
|
+
root.add_argument("--project", default=".")
|
|
1229
|
+
root.add_argument("--source", default="")
|
|
1230
|
+
root.add_argument("--engine", choices=["r", "python"], default="")
|
|
1231
|
+
root.add_argument("--mode", choices=["chapter", "figure"], default="")
|
|
1232
|
+
root.add_argument("--confirm-overwrite", action="store_true")
|
|
1233
|
+
root.add_argument("--stale-only", action="store_true")
|
|
1234
|
+
return root
|
|
1235
|
+
|
|
1236
|
+
|
|
1237
|
+
def main(argv: list[str] | None = None) -> int:
|
|
1238
|
+
args = parser().parse_args(argv)
|
|
1239
|
+
project = Path(args.project).expanduser().resolve()
|
|
1240
|
+
try:
|
|
1241
|
+
if args.command in {"status", "check"}:
|
|
1242
|
+
result = status(project, source=args.source, engine=args.engine, mode=args.mode)
|
|
1243
|
+
elif args.command == "render":
|
|
1244
|
+
result = render(project, source=args.source, engine=args.engine, mode=args.mode, confirm_overwrite=args.confirm_overwrite, stale_only=args.stale_only)
|
|
1245
|
+
elif args.command == "prune":
|
|
1246
|
+
result = prune(project)
|
|
1247
|
+
elif args.command == "resolve":
|
|
1248
|
+
if not args.engine:
|
|
1249
|
+
raise ComputationError("resolve requires --engine r or --engine python")
|
|
1250
|
+
config, _ = load_config(project)
|
|
1251
|
+
result = {**resolve_image(config, args.engine), "ok": True}
|
|
1252
|
+
else:
|
|
1253
|
+
if not args.engine:
|
|
1254
|
+
raise ComputationError("image requires --engine r or --engine python")
|
|
1255
|
+
config, _ = load_config(project)
|
|
1256
|
+
result = build_engine_image(project, config, args.engine)
|
|
1257
|
+
print(json_dump(result), end="")
|
|
1258
|
+
if args.command == "status":
|
|
1259
|
+
return 0
|
|
1260
|
+
return 0 if result.get("ok") else 1
|
|
1261
|
+
except ComputationError as exc:
|
|
1262
|
+
print(json_dump({"project": str(project), "ok": False, "error": str(exc)}), end="")
|
|
1263
|
+
return 2
|
|
1264
|
+
|
|
1265
|
+
|
|
1266
|
+
if __name__ == "__main__":
|
|
1267
|
+
raise SystemExit(main(sys.argv[1:]))
|