@feelpp/antora-extensions 1.0.0-rc.3 → 1.0.0-rc.5

package/package.json

@@ -1,7 +1,7 @@
 {
   "name": "@feelpp/antora-extensions",
   "private": false,
-  "version": "1.0.0-rc.3",
+  "version": "1.0.0-rc.5",
   "description": "Antora Extensions by Feel++.",
   "main": "src/index.js",
   "repository": {
@@ -20,11 +20,13 @@
   "homepage": "https://github.com/feelpp/feelpp-antora-extensions#readme",
   "devDependencies": {
     "chai": "^4.3.7",
-    "mocha": "^10.2.0",
     "libnpmpublish": "^4.0.2",
+    "mocha": "^10.2.0",
     "pacote": "^12.0.2"
   },
   "dependencies": {
+    "@asciidoctor/core": "2.2.9",
+    "asciidoctor-jupyter": "0.7.0",
     "jsdom": "^20.0.3"
   }
 }

package/src/index.js

@@ -1,9 +1,11 @@
 const {register: registerTooboxNavigation} = require('./toolbox-navigation.js')
 const {register: registerListing} = require('./listing.js')
 const {register: registerLunr} = require('./lunr.js')
+const {register: registerJupyter} = require('./jupyter.js')
 
 module.exports.register = (thisContext, args) => {
   registerTooboxNavigation.call(thisContext, args)
   registerListing.call(thisContext, args)
   registerLunr.call(thisContext, args)
+  registerJupyter.call(thisContext, args)
 }

package/src/jupyter.js

@@ -0,0 +1,277 @@
+const crypto = require('node:crypto')
+const path = require('node:path').posix
+
+const asciidoctor = require('@asciidoctor/core')()
+const JupyterConverter = require('asciidoctor-jupyter')
+const extensionPackage = require('../package.json')
+const converterPackage = require('asciidoctor-jupyter/package.json')
+
+asciidoctor.ConverterFactory.register(JupyterConverter, ['jupyter'])
+
+const DEFAULT_FIGURE_CONTRACT = 'alt text, caption, labelled axes and units, and non-colour encoding'
+
+function sha256 (value) {
+  return crypto.createHash('sha256').update(value).digest('hex')
+}
+
+function parseHeaderAttributes (source) {
+  const attributes = {}
+  const lines = source.split(/\r?\n/)
+  let titleSeen = false
+  for (const line of lines) {
+    if (!titleSeen) {
+      if (line.startsWith('= ')) titleSeen = true
+      continue
+    }
+    if (line === '' || line.startsWith('//')) continue
+    const match = line.match(/^:([^:]+):(?:\s*(.*))?$/)
+    if (!match) break
+    attributes[match[1]] = match[2] || ''
+  }
+  return attributes
+}
+
+function listAttribute (attributes, name) {
+  return (attributes[name] || '')
+    .split(',')
+    .map((value) => value.trim())
+    .filter(Boolean)
+}
+
+function resourceId ({ component, version, module, relative }, family = 'page') {
+  const prefix = version ? `${version}@${component}` : component
+  const familyMarker = family === 'page' ? '' : `${family}$`
+  return `${prefix}:${module}:${familyMarker}${relative}`
+}
+
+function matchesInclude (relative, include) {
+  if (!include || include.length === 0) return true
+  return include.some((pattern) => {
+    const escaped = pattern.replace(/[.+?^${}()|[\]\\]/g, '\\$&').replaceAll('*', '.*')
+    return new RegExp(`^${escaped}$`).test(relative)
+  })
+}
+
+function isJupyterPage (page, options = {}) {
+  const source = page.contents.toString('utf8')
+  const attributes = parseHeaderAttributes(source)
+  return Object.hasOwn(attributes, 'page-jupyter') && matchesInclude(page.src.relative, options.include)
+}
+
+function notebookMetadata (page, source, attributes, options) {
+  const required = [
+    'page-course-language',
+    'page-course-priority',
+    'page-course-difficulty',
+    'page-course-duration-minutes',
+    'page-course-concepts',
+    'page-course-outcomes'
+  ]
+  const missing = required.filter((name) => !attributes[name])
+  if (missing.length) {
+    throw new Error(`${resourceId(page.src)}: missing notebook metadata: ${missing.join(', ')}`)
+  }
+  if (attributes['page-course-language'] !== 'en') {
+    throw new Error(`${resourceId(page.src)}: generated notebooks must use English`)
+  }
+  const duration = Number.parseInt(attributes['page-course-duration-minutes'], 10)
+  if (!Number.isInteger(duration) || duration < 1) {
+    throw new Error(`${resourceId(page.src)}: invalid page-course-duration-minutes`)
+  }
+  return {
+    schema_version: 1,
+    source_kind: 'asciidoc',
+    source_resource_id: resourceId(page.src),
+    source_path: page.src.relative,
+    source_sha256: sha256(source),
+    source_page_url: page.pub.url,
+    language: 'en',
+    priority: attributes['page-course-priority'],
+    difficulty: attributes['page-course-difficulty'],
+    duration_minutes: duration,
+    concept_ids: listAttribute(attributes, 'page-course-concepts'),
+    outcomes: listAttribute(attributes, 'page-course-outcomes'),
+    execution_profile: attributes['page-course-execution-profile'] || 'fast',
+    accessibility: {
+      keyboard_only: true,
+      colour_alone_forbidden: true,
+      required_figure_contract: attributes['page-course-figure-contract'] || DEFAULT_FIGURE_CONTRACT
+    },
+    generator: {
+      asciidoctor_jupyter: converterPackage.version,
+      feelpp_antora_extensions: extensionPackage.version,
+      feelpp_asciidoctor_extensions: options.asciidoctorExtensionVersion || '1.0.0-rc.17'
+    }
+  }
+}
+
+function normaliseMarkdown (source, page) {
+  return source
+    .replace(
+      /(^|\n)\+\n(\\[\s\S]*?)\n\+(?=\n|$)/g,
+      (_match, prefix, latex) => `${prefix}$$\n${latex}\n$$`
+    )
+    .replace(/\]\(attachment\$([^)]+)\)/g, (_match, target) => {
+      const relative = path.relative(path.dirname(page.src.relative), target)
+      return `](${relative || path.basename(target)})`
+    })
+}
+
+function normaliseNotebook (notebook, page, source, attributes, options = {}) {
+  notebook.nbformat = 4
+  notebook.nbformat_minor = Math.max(notebook.nbformat_minor || 0, 5)
+  notebook.metadata = notebook.metadata || {}
+  notebook.metadata.kernelspec = notebook.metadata.kernelspec || {}
+  notebook.metadata.kernelspec.name = notebook.metadata.kernelspec.name || 'python3'
+  notebook.metadata.kernelspec.language = notebook.metadata.kernelspec.language || 'python'
+  notebook.metadata.kernelspec.display_name = notebook.metadata.kernelspec.display_name || 'Python 3'
+  notebook.metadata.course = notebookMetadata(page, source, attributes, options)
+  notebook.cells.forEach((cell, index) => {
+    let cellSource = Array.isArray(cell.source) ? cell.source.join('') : cell.source || ''
+    if (cell.cell_type === 'markdown') {
+      cellSource = normaliseMarkdown(cellSource, page)
+      cell.source = cellSource
+    }
+    cell.id = sha256(`${resourceId(page.src)}\0${index}\0${cell.cell_type}\0${cellSource}`).slice(0, 16)
+    if (cell.cell_type === 'code') {
+      cell.execution_count = null
+      cell.outputs = []
+    }
+  })
+  return notebook
+}
+
+function generateNotebook (page, options = {}) {
+  const source = page.contents.toString('utf8')
+  const attributes = parseHeaderAttributes(source)
+  let notebook
+  try {
+    notebook = JSON.parse(asciidoctor.convert(source, {
+      backend: 'jupyter',
+      safe: 'safe',
+      standalone: true,
+      to_file: false,
+      attributes: {
+        'jupyter-language-name': attributes['jupyter-language-name'] || 'python',
+        'jupyter-language-version': attributes['jupyter-language-version'] || '3.12',
+        'jupyter-kernel-name': attributes['jupyter-kernel-name'] || 'python3',
+        'jupyter-kernel-language': attributes['jupyter-kernel-language'] || 'python'
+      }
+    }))
+  } catch (error) {
+    throw new Error(`${resourceId(page.src)}: notebook conversion failed: ${error.message}`)
+  }
+  normaliseNotebook(notebook, page, source, attributes, options)
+  return Buffer.from(`${JSON.stringify(notebook, null, 2)}\n`)
+}
+
+function attachmentRelative (page) {
+  return page.src.relative.replace(/\.adoc$/, '.ipynb')
+}
+
+function manifestEntry (page, attachment, notebook) {
+  const metadata = notebook.metadata.course
+  const codeBlocks = notebook.cells
+    .map((cell, index) => ({ cell, index }))
+    .filter(({ cell }) => cell.cell_type === 'code')
+    .map(({ cell, index }) => ({
+      id: cell.id,
+      index,
+      source_sha256: sha256(Array.isArray(cell.source) ? cell.source.join('') : cell.source || ''),
+      dynamic: false,
+      required: true
+    }))
+  return {
+    source_resource_id: metadata.source_resource_id,
+    source_path: metadata.source_path,
+    source_sha256: metadata.source_sha256,
+    page_url: page.pub.url,
+    notebook_resource_id: resourceId(attachment.src, 'attachment'),
+    notebook_path: attachment.src.relative,
+    notebook_sha256: sha256(attachment.contents),
+    notebook_url: attachment.pub.url,
+    priority: metadata.priority,
+    difficulty: metadata.difficulty,
+    concept_ids: metadata.concept_ids,
+    outcomes: metadata.outcomes,
+    code_blocks: codeBlocks,
+    warnings: []
+  }
+}
+
+function buildManifest (entries) {
+  return Buffer.from(`${JSON.stringify({
+    schema_version: 1,
+    artifact: 'asciidoc-notebook-manifest',
+    generator: {
+      name: extensionPackage.name,
+      version: extensionPackage.version
+    },
+    entries: [...entries].sort((a, b) => a.source_resource_id.localeCompare(b.source_resource_id))
+  }, null, 2)}\n`)
+}
+
+function register ({ config = {} } = {}) {
+  const logger = this.getLogger('jupyter-extension')
+  const options = config.jupyter || {}
+  const generated = new Map()
+
+  this.on('contentClassified', ({ contentCatalog }) => {
+    const pages = contentCatalog.getPages().filter((page) => isJupyterPage(page, options))
+      .sort((a, b) => resourceId(a.src).localeCompare(resourceId(b.src)))
+    const entriesByComponent = new Map()
+    for (const page of pages) {
+      const contents = generateNotebook(page, options)
+      const attachment = contentCatalog.addFile({
+        contents,
+        src: {
+          component: page.src.component,
+          version: page.src.version,
+          module: page.src.module,
+          family: 'attachment',
+          relative: attachmentRelative(page)
+        }
+      })
+      const notebook = JSON.parse(contents.toString('utf8'))
+      const entry = manifestEntry(page, attachment, notebook)
+      const key = `${page.src.component}\0${page.src.version || ''}`
+      const group = entriesByComponent.get(key) || { page, entries: [] }
+      group.entries.push(entry)
+      entriesByComponent.set(key, group)
+      generated.set(page, attachment)
+      logger.info(`generated ${attachment.pub.url} from ${resourceId(page.src)}`)
+    }
+    for (const { page, entries } of entriesByComponent.values()) {
+      contentCatalog.addFile({
+        contents: buildManifest(entries),
+        src: {
+          component: page.src.component,
+          version: page.src.version,
+          module: 'ROOT',
+          family: 'attachment',
+          relative: 'generated/asciidoc-notebook-manifest.json'
+        }
+      })
+    }
+  })
+
+  this.on('documentsConverted', () => {
+    for (const [page, attachment] of generated) {
+      page.asciidoc.attributes['page-jupyter-url'] = attachment.pub.url
+      page.asciidoc.attributes['page-jupyter-status'] = 'generated'
+    }
+  })
+}
+
+module.exports = {
+  attachmentRelative,
+  buildManifest,
+  generateNotebook,
+  isJupyterPage,
+  manifestEntry,
+  normaliseMarkdown,
+  parseHeaderAttributes,
+  register,
+  resourceId,
+  sha256
+}

package/src/listing.js

@@ -10,7 +10,7 @@ const template = require('./template')
 /**
  * @module listing-extension
  */
-function register ({ config: { listing, ...unknownOptions } }) {
+function register ({ config: { listing, jupyter: _jupyter, lunr: _lunr, ...unknownOptions } }) {
   const packageName = "@feelpp/antora-listing-extension"
   const logger = this.getLogger(packageName)
 

package/tests/fixtures/jupyter-contract/content/antora.yml

@@ -0,0 +1,5 @@
+name: executable-contract
+title: Executable contract fixture
+version: ~
+nav:
+  - modules/ROOT/nav.adoc

package/tests/fixtures/jupyter-contract/content/modules/ROOT/nav.adoc

@@ -0,0 +1,2 @@
+* xref:foundations/sample.adoc[Generated notebook sample]
+* xref:plain.adoc[Plain page]

package/tests/fixtures/jupyter-contract/content/modules/ROOT/pages/foundations/sample.adoc

@@ -0,0 +1,28 @@
+= Generated notebook sample
+:page-jupyter:
+:page-course-language: en
+:page-course-concepts: DP-FND-01, DP-REP-01
+:page-course-priority: P0
+:page-course-difficulty: D1
+:page-course-duration-minutes: 20
+:page-course-outcomes: LO1, LO9
+:page-course-execution-profile: fast
+:jupyter-language-name: python
+:jupyter-language-version: 3.12
+:jupyter-kernel-name: python3
+:jupyter-kernel-language: python
+
+The page is the canonical explanation. Its generated notebook preserves this prose and the executable evidence below.
+
+[stem]
+++++
+\widehat\mu = \frac{1}{n}\sum_{i=1}^{n}X_i.
+++++
+
+[source,python]
+----
+value = 6 * 7
+print(value)
+----
+
+xref:attachment$foundations/sample.ipynb[Download the generated notebook].

package/tests/fixtures/jupyter-contract/content/modules/ROOT/pages/plain.adoc

@@ -0,0 +1,3 @@
+= Plain page
+
+This page deliberately does not request a notebook.

package/tests/fixtures/jupyter-contract/expected.json

@@ -0,0 +1,16 @@
+{
+  "schema_version": 1,
+  "component": "executable-contract",
+  "version": "",
+  "module": "ROOT",
+  "source_relative_path": "foundations/sample.adoc",
+  "attachment_relative_path": "foundations/sample.ipynb",
+  "page_url": "/executable-contract/foundations/sample.html",
+  "attachment_url": "/executable-contract/_attachments/foundations/sample.ipynb",
+  "plain_page_generates_notebook": false,
+  "ui_contract": {
+    "requested_attribute": "jupyter",
+    "resolved_url_attribute": "jupyter-url",
+    "status_attribute": "jupyter-status"
+  }
+}

package/tests/fixtures/jupyter-contract/failures.json

@@ -0,0 +1,30 @@
+{
+  "schema_version": 1,
+  "cases": [
+    {
+      "id": "duplicate-attachment",
+      "importance": "P0",
+      "expected": "reject two generated resources with the same Antora attachment ID"
+    },
+    {
+      "id": "converter-warning",
+      "importance": "P0",
+      "expected": "reject a required page when the converter reports an unsupported node"
+    },
+    {
+      "id": "missing-converter",
+      "importance": "P0",
+      "expected": "fail with the package name and requested page resource ID"
+    },
+    {
+      "id": "unresolved-url",
+      "importance": "P0",
+      "expected": "do not expose a notebook URL or generated status"
+    },
+    {
+      "id": "nondeterministic-manifest",
+      "importance": "P1",
+      "expected": "reject timestamps, unstable ordering, or missing source/output digests"
+    }
+  ]
+}

package/tests/jupyter-contract-test.js

@@ -0,0 +1,152 @@
+const fs = require('node:fs')
+const path = require('node:path')
+const chai = require('chai')
+const expect = chai.expect
+const jupyter = require('../src/jupyter.js')
+
+const fixtureDir = path.join(__dirname, 'fixtures', 'jupyter-contract')
+
+describe('Jupyter generation contract fixture', () => {
+  it('defines a page-relative attachment and confirmed UI URL contract', () => {
+    const expected = JSON.parse(fs.readFileSync(path.join(fixtureDir, 'expected.json'), 'utf8'))
+
+    expect(expected.source_relative_path).to.equal('foundations/sample.adoc')
+    expect(expected.attachment_relative_path).to.equal('foundations/sample.ipynb')
+    expect(expected.attachment_url).to.equal(
+      '/executable-contract/_attachments/foundations/sample.ipynb'
+    )
+    expect(expected.plain_page_generates_notebook).to.equal(false)
+    expect(expected.ui_contract.resolved_url_attribute).to.equal('jupyter-url')
+    expect(expected.ui_contract.status_attribute).to.equal('jupyter-status')
+  })
+
+  it('contains marked and unmarked representative Antora pages', () => {
+    const pages = path.join(fixtureDir, 'content', 'modules', 'ROOT', 'pages')
+    const marked = fs.readFileSync(path.join(pages, 'foundations', 'sample.adoc'), 'utf8')
+    const plain = fs.readFileSync(path.join(pages, 'plain.adoc'), 'utf8')
+
+    expect(marked).to.contain(':page-jupyter:')
+    expect(marked).to.contain('[source,python]')
+    expect(plain).not.to.contain(':page-jupyter:')
+  })
+
+  it('inventories required generation failures before implementation', () => {
+    const failures = JSON.parse(fs.readFileSync(path.join(fixtureDir, 'failures.json'), 'utf8'))
+    const ids = failures.cases.map((item) => item.id)
+
+    expect(ids).to.deep.equal([
+      'duplicate-attachment',
+      'converter-warning',
+      'missing-converter',
+      'unresolved-url',
+      'nondeterministic-manifest'
+    ])
+    expect(failures.cases.filter((item) => item.importance === 'P0')).to.have.length(4)
+  })
+
+  it('converts only marked pages and normalises notebook metadata deterministically', () => {
+    const pagesRoot = path.join(fixtureDir, 'content', 'modules', 'ROOT', 'pages')
+    const page = fixturePage('foundations/sample.adoc', pagesRoot)
+    const plain = fixturePage('plain.adoc', pagesRoot)
+
+    expect(jupyter.isJupyterPage(page)).to.equal(true)
+    expect(jupyter.isJupyterPage(plain)).to.equal(false)
+    const first = jupyter.generateNotebook(page)
+    const second = jupyter.generateNotebook(page)
+    expect(first.equals(second)).to.equal(true)
+
+    const notebook = JSON.parse(first.toString('utf8'))
+    expect(notebook.nbformat).to.equal(4)
+    expect(notebook.nbformat_minor).to.equal(5)
+    expect(notebook.metadata.kernelspec).to.include({
+      name: 'python3',
+      language: 'python',
+      display_name: 'Python 3'
+    })
+    expect(notebook.metadata.course).to.include({
+      source_kind: 'asciidoc',
+      source_path: 'foundations/sample.adoc',
+      priority: 'P0',
+      difficulty: 'D1',
+      language: 'en',
+      execution_profile: 'fast'
+    })
+    expect(notebook.metadata.course.concept_ids).to.deep.equal(['DP-FND-01', 'DP-REP-01'])
+    expect(notebook.cells.every((cell) => /^[0-9a-f]{16}$/.test(cell.id))).to.equal(true)
+    const markdown = notebook.cells
+      .filter((cell) => cell.cell_type === 'markdown')
+      .map((cell) => Array.isArray(cell.source) ? cell.source.join('') : cell.source)
+      .join('\n')
+    expect(markdown).to.contain('$$\n\\widehat\\mu')
+    expect(markdown).to.contain('](sample.ipynb)')
+    expect(markdown).not.to.match(/\n\+\n\\widehat/)
+    expect(markdown).not.to.contain('attachment$')
+    expect(notebook.cells.filter((cell) => cell.cell_type === 'code'))
+      .to.satisfy((cells) => cells.length === 1 && cells.every((cell) => cell.outputs.length === 0))
+  })
+
+  it('registers page-relative attachments, a manifest, and confirmed UI attributes', () => {
+    const pagesRoot = path.join(fixtureDir, 'content', 'modules', 'ROOT', 'pages')
+    const page = fixturePage('foundations/sample.adoc', pagesRoot)
+    const plain = fixturePage('plain.adoc', pagesRoot)
+    const files = []
+    const contentCatalog = {
+      getPages: () => [plain, page],
+      addFile: (file) => {
+        if (files.some((candidate) => JSON.stringify(candidate.src) === JSON.stringify(file.src))) {
+          throw new Error(`duplicate attachment: ${file.src.relative}`)
+        }
+        file.pub = {
+          url: `/executable-contract/_attachments/${file.src.relative}`
+        }
+        files.push(file)
+        return file
+      }
+    }
+    const handlers = {}
+    const context = {
+      getLogger: () => ({ info: () => {} }),
+      on: (event, handler) => { handlers[event] = handler }
+    }
+
+    jupyter.register.call(context, { config: {} })
+    handlers.contentClassified({ contentCatalog })
+    page.asciidoc = { attributes: {} }
+    plain.asciidoc = { attributes: {} }
+    handlers.documentsConverted()
+
+    expect(files.map((file) => file.src.relative)).to.deep.equal([
+      'foundations/sample.ipynb',
+      'generated/asciidoc-notebook-manifest.json'
+    ])
+    expect(page.asciidoc.attributes).to.include({
+      'page-jupyter-status': 'generated',
+      'page-jupyter-url': '/executable-contract/_attachments/foundations/sample.ipynb'
+    })
+    expect(plain.asciidoc.attributes).to.deep.equal({})
+
+    const manifest = JSON.parse(files[1].contents.toString('utf8'))
+    expect(manifest.entries).to.have.length(1)
+    expect(manifest.entries[0].notebook_path).to.equal('foundations/sample.ipynb')
+    expect(manifest.entries[0].notebook_url).to.equal(
+      '/executable-contract/_attachments/foundations/sample.ipynb'
+    )
+    expect(manifest).not.to.have.property('generated_at')
+  })
+})
+
+function fixturePage (relative, root) {
+  return {
+    contents: fs.readFileSync(path.join(root, relative)),
+    src: {
+      component: 'executable-contract',
+      version: '',
+      module: 'ROOT',
+      family: 'page',
+      relative
+    },
+    pub: {
+      url: `/executable-contract/${relative.replace(/\.adoc$/, '.html')}`
+    }
+  }
+}