@uniweb/build 0.14.15 → 0.14.17

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@uniweb/build",
-  "version": "0.14.15",
+  "version": "0.14.17",
   "description": "Build tooling for the Uniweb Component Web Platform",
   "type": "module",
   "exports": {
@@ -63,9 +63,9 @@
     "@uniweb/theming": "0.1.4"
   },
   "optionalDependencies": {
-    "@uniweb/runtime": "0.8.19",
     "@uniweb/content-reader": "1.1.12",
-    "@uniweb/schemas": "0.2.3"
+    "@uniweb/schemas": "0.2.3",
+    "@uniweb/runtime": "0.8.20"
   },
   "peerDependencies": {
     "vite": "^5.0.0 || ^6.0.0 || ^7.0.0",

package/src/site/data-ball.js

@@ -0,0 +1,122 @@
+/**
+ * Data ball — the static-delivery half of a composite `uniweb deploy`. A site's
+ * collections partition by schema presence: a collection that resolves a data schema
+ * syncs as folder entities; a SCHEMA-LESS collection has no entity model, so its built
+ * `dist/data/<name>.json` (cascade + any `deferred:` per-record files) is delivered
+ * statically. This bundles that schema-less subset of `dist/data/**` plus the whole
+ * `dist/_search/**` index into one JSON doc the deploy uploads as a single
+ * content-addressed asset; the backend unwraps it into the `/data/*` + `/_search/*`
+ * bytes the gateway serves.
+ *
+ *   { data:   { "<relpath-under-data>":    <json> },   // schema-less collections only
+ *     search: { "<relpath-under-_search>": <json> } }  // the whole (baked) index
+ *
+ * Search is NOT filtered: the index is baked over all content (the live/baked seam —
+ * schema-backed lists are served live from entities, but their search entries are baked
+ * here until the next deploy).
+ */
+
+import { existsSync } from 'node:fs'
+import { readFile, readdir } from 'node:fs/promises'
+import { join, relative, sep } from 'node:path'
+import { isLocalAssetPath } from './assets.js'
+
+// Walk a dist subdir for *.json → { "<posix-relpath>": <parsedJson> }. Unparseable
+// files are skipped (build-emitted data is always valid JSON; this just stays safe).
+async function readJsonTree(dir) {
+  if (!existsSync(dir)) return {}
+  const out = {}
+  const entries = await readdir(dir, { withFileTypes: true, recursive: true })
+  for (const entry of entries) {
+    if (!entry.isFile() || !entry.name.endsWith('.json')) continue
+    const full = join(entry.parentPath || entry.path, entry.name)
+    const rel = relative(dir, full).split(sep).join('/')
+    try {
+      out[rel] = JSON.parse(await readFile(full, 'utf8'))
+    } catch {
+      // not valid JSON — skip
+    }
+  }
+  return out
+}
+
+// The collection a `dist/data` relpath belongs to: the first path segment, minus a
+// trailing `.json`. `articles.json` → `articles`; `articles/hello.json` → `articles`.
+function collectionOf(relPath) {
+  const first = relPath.split('/')[0]
+  return first.endsWith('.json') ? first.slice(0, -5) : first
+}
+
+/**
+ * Assemble the static-data ball from a built site's dist/.
+ *
+ * @param {string} distDir - the site's built dist/ directory
+ * @param {string[]} [schemalessNames] - collection names with no data schema (from
+ *        `emitSyncPackages(...).schemaless`); only these contribute `data`.
+ * @returns {Promise<{ data: Object, search: Object }|null>} null when there is nothing
+ *          to deliver (no schema-less data AND no search index).
+ */
+export async function assembleDataBall(distDir, schemalessNames = []) {
+  const schemaless = new Set(schemalessNames)
+  const allData = await readJsonTree(join(distDir, 'data'))
+  const data = {}
+  for (const [relPath, value] of Object.entries(allData)) {
+    if (schemaless.has(collectionOf(relPath))) data[relPath] = value
+  }
+  const search = await readJsonTree(join(distDir, '_search'))
+  if (Object.keys(data).length === 0 && Object.keys(search).length === 0) return null
+  return { data, search }
+}
+
+// --- local media in the ball -------------------------------------------------
+// Schema-less collection data rides in the ball, so a local image in a schema-less
+// record (e.g. a note's `image: /images/x.png`) needs the same upload + serve-URL
+// rewrite the entity content gets (emitSyncPackages' `assetRewrite`) — otherwise the
+// served `/data/<name>.json` keeps a dangling local path. The deploy collects the
+// ball's refs, uploads them on the SAME asset lane, then rewrites the ball before
+// uploading it. The backend serves a `serve_url` in the ball identically to one in an
+// entity (it unwraps the ball verbatim), so this is purely producer-side.
+
+/**
+ * Site-root local asset refs anywhere in the ball (`/images/x.png`, `/collections/...`).
+ * Built `dist/data` refs are already site-root (the collection processor copied
+ * co-located assets to `public/collections/**`), so only `/`-prefixed refs are collected.
+ * @param {{data:object,search:object}|null} ball
+ * @returns {string[]} deduped refs to upload
+ */
+export function collectBallAssets(ball) {
+  const refs = new Set()
+  const walk = (n) => {
+    if (typeof n === 'string') {
+      if (isLocalAssetPath(n) && n.startsWith('/')) refs.add(n)
+      return
+    }
+    if (Array.isArray(n)) { for (const x of n) walk(x); return }
+    if (n && typeof n === 'object') for (const v of Object.values(n)) walk(v)
+  }
+  walk(ball)
+  return [...refs]
+}
+
+/**
+ * Rewrite the ball: replace every local ref the map covers with its serve URL. Pure —
+ * returns a NEW ball (the input is reused elsewhere). A ref the map omits (upload
+ * failed/skipped) is left untouched — never a broken URL.
+ * @param {{data:object,search:object}|null} ball
+ * @param {Record<string,string>} map - ref → serve URL
+ * @returns {{data:object,search:object}|null} a new ball, or the input when there's nothing to do
+ */
+export function rewriteBallAssets(ball, map) {
+  if (!ball || !map || Object.keys(map).length === 0) return ball
+  const walk = (n) => {
+    if (typeof n === 'string') return map[n] || n
+    if (Array.isArray(n)) return n.map(walk)
+    if (n && typeof n === 'object') {
+      const out = {}
+      for (const [k, v] of Object.entries(n)) out[k] = walk(v)
+      return out
+    }
+    return n
+  }
+  return walk(ball)
+}

package/src/site/index.js

@@ -38,6 +38,7 @@ export {
   writeCollectionFiles,
   getCollectionLastModified
 } from './collection-processor.js'
+export { assembleDataBall, collectBallAssets, rewriteBallAssets } from './data-ball.js'
 export {
   parseFetchConfig,
   executeFetch,

package/src/uwx/collections.js

@@ -422,17 +422,20 @@ async function loadSourceRecords(siteRoot, decl) {
  *        `@uniweb/data-schema` declaration (or null). The verb wires this to the
  *        backend's Model-read route. Without it, the local foundation is required.
  * @param {string} [opts.sourceLocale]    - localized-field wrap locale
- * @returns {Promise<{ entities: object[], index: object[], warnings: string[], mappedCount: number }>}
+ * @returns {Promise<{ entities: object[], index: object[], warnings: string[], schemaless: Array<{name: string}>, mappedCount: number }>}
+ *   `schemaless` lists collections that resolved no data schema (the convention-
+ *   default soft-skip) — not synced as entities; the composite deploy delivers
+ *   them statically via the data ball.
  */
 export async function buildCollectionEntities(siteRoot, opts = {}) {
   // Merged collections config (collections.yml over site.yml::collections). Reused
   // from the caller when provided (sync-package shares it with the folder builder).
   const colConfig = opts.collectionsConfig || (await resolveCollectionsConfig(siteRoot))
   if (!colConfig.folderSync) {
-    return { entities: [], index: [], warnings: [], mappedCount: 0, colConfig }
+    return { entities: [], index: [], warnings: [], schemaless: [], mappedCount: 0, colConfig }
   }
   const mapped = syncableCollections(colConfig.declarations)
-  if (mapped.length === 0) return { entities: [], index: [], warnings: [], mappedCount: 0, colConfig }
+  if (mapped.length === 0) return { entities: [], index: [], warnings: [], schemaless: [], mappedCount: 0, colConfig }
 
   // A Model declaration comes from a LOCAL foundation (offline) or, for a
   // non-local Model, from the injected async `resolveModel(name)` — the verb wires
@@ -474,6 +477,10 @@ export async function buildCollectionEntities(siteRoot, opts = {}) {
   const entities = []
   const index = []
   const warnings = []
+  // Collections that resolved no data schema (the convention-default soft-skip
+  // below) — not synced as folder entities. The composite deploy delivers these
+  // statically (the "data ball") instead, so the caller can route them there.
+  const schemaless = []
   // The sync response is keyed per ($model, $id), so the pair must be unique
   // within one submission (two collections on the same Model could otherwise
   // reuse a slug).
@@ -489,6 +496,7 @@ export async function buildCollectionEntities(siteRoot, opts = {}) {
         warnings.push(
           `${name}: no data schema "${modelName}" (subfolder-name default) — not synced`
         )
+        schemaless.push({ name })
         continue
       }
       throw new Error(
@@ -577,7 +585,7 @@ export async function buildCollectionEntities(siteRoot, opts = {}) {
     warnings.push(...mappedOut.warnings)
   }
 
-  return { entities, index, warnings, mappedCount: mapped.length, colConfig }
+  return { entities, index, warnings, schemaless, mappedCount: mapped.length, colConfig }
 }
 
 /**

package/src/uwx/site-project.js

@@ -101,6 +101,7 @@ const INFO_TO_SITE_YML = {
   data: 'data',
   template: 'template',
   seo: 'seo',
+  app: 'app', // deployment's @uniweb/app-spec ref (bare uuid; deployment-local)
 }
 
 /**

package/src/uwx/site.js

@@ -571,6 +571,11 @@ export async function siteProjectToDocument(siteRoot, opts = {}) {
   setIf(info, 'search', siteYml.search)
   setIf(info, 'paths', siteYml.paths)
   setIf(info, 'data', siteYml.data ?? siteYml.fetch)
+  // `app` — the deployment's `@uniweb/app-spec` reference (a bare uuid string),
+  // bound when the site is hosted as a composite. Authored config that round-trips
+  // verbatim/opaque (like `foundation`), so a pull→edit→push preserves it; it is
+  // deployment-LOCAL (not portable across deployments). (uwx-format.md → info.app.)
+  setIf(info, 'app', siteYml.app)
   // `template: true` designates this site as a clonable SITE-TEMPLATE: on push the
   // backend applies a clonability designation to this site-content entity (it is
   // NOT a registry artifact). Verbatim; absent → a normal (non-template) site.

package/src/uwx/sync-package.js

@@ -23,14 +23,15 @@ import { buildCollectionEntities, entityContentHash } from './collections.js'
 import { buildFolderEntity } from './folder.js'
 import { siteProjectToDocument } from './site.js'
 import { emitEntitySyncPackage } from './entity-document.js'
+import { isLocalAssetPath } from '../site/assets.js'
 
 const SITE_MODEL_NAME = '@uniweb/site-content'
 const SITE_ENTITY_KEY = 'site-content'
 
 const cacheKey = (entity) => `${entity.model} ${entity.id}`
 
-function emitLane(entities, exporter, exportedAt) {
-  const models = [...new Set(entities.map((e) => e.model))]
+function emitLane(entities, exporter, exportedAt, extraModels = []) {
+  const models = [...new Set([...entities.map((e) => e.model), ...extraModels])]
   const buffer = emitEntitySyncPackage({
     entities,
     modelsRequired: models.map((name) => ({ name_at_export: name })),
@@ -40,6 +41,68 @@ function emitLane(entities, exporter, exportedAt) {
   return { buffer, entityCount: entities.length, models }
 }
 
+// Collect the Models referenced by the folder's `ref` leaves (`entry.model`), walking
+// the contents/$children tree. The folder is built from the FULL record set, so it
+// references every record's Model — including records the send-only-changed filter
+// drops from THIS package (a re-push where the folder changed but the records didn't).
+// The backend requires every referenced Model declared in modelsRequired, so these must
+// ride even when their record entities don't.
+function collectReferencedModels(node, acc) {
+  if (!node || typeof node !== 'object') return acc
+  if (node.entry && typeof node.entry.model === 'string') acc.add(node.entry.model)
+  for (const key of ['$children', 'contents']) {
+    if (Array.isArray(node[key])) for (const child of node[key]) collectReferencedModels(child, acc)
+  }
+  return acc
+}
+
+// --- local-media over push (Slice 5) ----------------------------------------
+// A site's content references local media by the author's original path
+// (`/images/hero.png`, `./hero.png`); the backend stores media content-addressed
+// and serves it by URL. The deploy uploads the local files and swaps the refs for
+// the backend's serve URLs. Both steps walk the produced entity documents with one
+// generic recursion, filtered by `isLocalAssetPath` (a `/`/`./`/`../` prefix PLUS a
+// media extension) — so it covers PM image `src`, `background`/`params`, record
+// media fields, and localized `{locale: doc}` maps uniformly and safely (non-media
+// strings like `/data/x.json` never match). Mirrors the build's `walkDataAssets`.
+
+// Invoke visitor(ref) for every local asset path string anywhere in the document.
+function walkEntityAssets(node, visitor) {
+  if (typeof node === 'string') {
+    if (isLocalAssetPath(node)) visitor(node)
+    return
+  }
+  if (Array.isArray(node)) {
+    for (const item of node) walkEntityAssets(item, visitor)
+    return
+  }
+  if (node && typeof node === 'object') {
+    for (const v of Object.values(node)) walkEntityAssets(v, visitor)
+  }
+}
+
+// In-place: replace every local-asset-path string the map covers with its serve
+// URL. A ref the map omits (upload failed/skipped) is left untouched — never a
+// broken URL. Returns the (mutated) node.
+function rewriteEntityAssets(node, map) {
+  if (Array.isArray(node)) {
+    for (let i = 0; i < node.length; i++) {
+      const v = node[i]
+      if (typeof v === 'string') { if (map[v]) node[i] = map[v] }
+      else rewriteEntityAssets(v, map)
+    }
+    return node
+  }
+  if (node && typeof node === 'object') {
+    for (const key of Object.keys(node)) {
+      const v = node[key]
+      if (typeof v === 'string') { if (map[v]) node[key] = map[v] }
+      else rewriteEntityAssets(v, map)
+    }
+  }
+  return node
+}
+
 /**
  * Build the two sync packages for a site.
  *
@@ -51,11 +114,22 @@ function emitLane(entities, exporter, exportedAt) {
  * @param {Object<string,string>} [opts.priorHashes] - sync-cache (send-only-changed)
  * @param {boolean} [opts.sendAll]        - bypass the prior-hash filter
  * @param {boolean} [opts.includeSite]    - include the site-content lane (default true)
+ * @param {object} [opts.injectInfo]      - deploy-derived `info.*` to stamp on the
+ *        site-content document (e.g. `{ data_bundle }`, the static-data ball URL);
+ *        wire-only — never authored in site.yml, never projected back on pull.
+ * @param {Object<string,string>} [opts.assetRewrite] - map of local asset ref →
+ *        backend serve URL; rewrites the entities' media refs before push (the
+ *        deploy's 2nd emit). Absent → no rewrite (the f225 sync path is unchanged).
  * @param {object} [opts.exporter] @param {string} [opts.exportedAt]
  * @returns {Promise<{
  *   siteContent: { buffer, entityCount, index, models }|null,
  *   collections: { buffer, entityCount, index, models }|null,
- *   hashes: Object<string,string>, warnings: string[], skipped: number }>}
+ *   hashes: Object<string,string>, warnings: string[], skipped: number,
+ *   schemaless: Array<{name: string}>, localAssets: string[] }>}
+ *   `schemaless` lists collections that resolved no data schema (soft-skipped from
+ *   the sync) — the composite deploy delivers these statically via the data ball.
+ *   `localAssets` lists the site-root local media refs (`/images/x.png`) the deploy
+ *   must upload + rewrite to serve URLs; co-located refs are warned and skipped.
  *   Each lane is null when it has nothing to push. The collections `index` keeps a
  *   leading `{ kind: 'folder' }` placeholder (submission position 0 → the folder
  *   entity) so record back-fill stays positionally aligned; the folder itself has no
@@ -84,6 +158,45 @@ export async function emitSyncPackages(siteRoot, opts = {}) {
   })
 
   const siteDoc = includeSite ? await siteProjectToDocument(siteRoot, { sourceLocale }) : null
+  // Deploy-derived `info` fields (e.g. `data_bundle`, the static-data ball URL) are
+  // stamped here — NOT authored in site.yml, so they ride the wire but never project
+  // back on pull (the `info.assets` precedent). They are part of the hashed content,
+  // so a changed bundle URL correctly re-fires the site-content lane.
+  if (siteDoc && opts.injectInfo && typeof opts.injectInfo === 'object') {
+    siteDoc.info = { ...siteDoc.info, ...opts.injectInfo }
+  }
+
+  // Local-media over push (Slice 5). `assetRewrite` ({ '/images/x.png': serveUrl })
+  // is supplied by the deploy's SECOND emit, after it has uploaded the files the
+  // FIRST emit surfaced in `localAssets`. It is absent on the collect emit and on
+  // every non-deploy caller, so the f225 sync path is byte-identical without it.
+  const assetRewrite =
+    opts.assetRewrite && typeof opts.assetRewrite === 'object' ? opts.assetRewrite : null
+  if (assetRewrite) {
+    if (siteDoc) rewriteEntityAssets(siteDoc, assetRewrite)
+    for (const e of col.entities) rewriteEntityAssets(e.document, assetRewrite)
+  }
+  // Collect the site-root local refs the deploy must upload (`/images/x.png`).
+  // Co-located refs (`./x`, `../x`) need the source `.md` location to resolve — the
+  // entity doesn't carry it — so warn once each and skip (v1: use a site-root path).
+  const localAssetSet = new Set()
+  const colocatedSeen = new Set()
+  const collectFrom = (doc) =>
+    doc &&
+    walkEntityAssets(doc, (ref) => {
+      if (ref.startsWith('/')) localAssetSet.add(ref)
+      else if (!colocatedSeen.has(ref)) {
+        colocatedSeen.add(ref)
+        warnings.push(
+          `local-media: co-located asset "${ref}" is not uploaded on the composite deploy — ` +
+            `use a site-root path (e.g. /images/${ref.replace(/^[./]+/, '')})`
+        )
+      }
+    })
+  collectFrom(siteDoc)
+  for (const e of col.entities) collectFrom(e.document)
+  const localAssets = [...localAssetSet]
+
   const siteEntity = siteDoc
     ? { id: siteDoc.$id, model: siteDoc.$model, file: 'entities/site-content.json', document: siteDoc }
     : null
@@ -114,7 +227,11 @@ export async function emitSyncPackages(siteRoot, opts = {}) {
     // back-fill (backfillEntityUuids skips it — the folder has no uuid to write).
     const entities = [folder, ...changedRecords.map((r) => r.entity)]
     const index = [{ kind: 'folder' }, ...changedRecords.map((r) => r.index)]
-    collections = { ...emitLane(entities, exporter, exportedAt), index }
+    // The folder references every record's Model via `entry.model` — including records
+    // filtered out here by send-only-changed. Declare them all (the backend rejects a
+    // folder that references an undeclared Model).
+    const referencedModels = [...collectReferencedModels(folder.document, new Set())]
+    collections = { ...emitLane(entities, exporter, exportedAt, referencedModels), index }
   }
 
   // --- site-content lane -------------------------------------------------------
@@ -128,5 +245,5 @@ export async function emitSyncPackages(siteRoot, opts = {}) {
   // the content lane by its presence/absence.
   const siteContentUuid = siteDoc?.$uuid
 
-  return { siteContent, collections, siteContentUuid, hashes, warnings, skipped }
+  return { siteContent, collections, siteContentUuid, hashes, warnings, skipped, schemaless: col.schemaless, localAssets }
 }

package/src/uwx/zip.js

@@ -1,11 +1,12 @@
 // Minimal, zero-dependency ZIP writer/reader for .uwx containers.
 //
-// DESIGN DECISION: Stored only (compression method 0), no Deflate. A .uwx
-// container is a ZIP; compression is an optimization, not part of the
-// contract, and every standard ZIP reader handles Stored entries. Staying
-// Stored-only removes a class of cross-tool byte asymmetry and needs no
-// zlib. If package size ever matters, a Deflate path can be added later via
-// Node's built-in `zlib.deflateRawSync` without changing callers.
+// DESIGN DECISION: our WRITER (`createZip`) emits Stored only (compression
+// method 0), no Deflate. A .uwx container is a ZIP; compression is an
+// optimization, not part of the contract, and every standard ZIP reader
+// handles Stored entries. Staying Stored-only on write removes a class of
+// cross-tool byte asymmetry. The READER (`readZip`) additionally inflates
+// Deflate (method 8) entries — the backend's pull `.uwx` Deflates larger
+// entities, and the framework must read what the backend produces.
 //
 // No Zip64: per-record JSON files are far below 4 GiB and entry counts far
 // below 65535. The format is otherwise the classic APPNOTE layout, all
@@ -17,6 +18,7 @@
 // makes byte output reproducible for a given input.
 
 import { crc32 } from './crc32.js'
+import { inflateRawSync } from 'node:zlib'
 
 const LOCAL_SIG = 0x04034b50
 const CENTRAL_SIG = 0x02014b50
@@ -89,11 +91,12 @@ export function createZip(files) {
 }
 
 /**
- * Reader for our own Stored archives — used by tests and by inspection /
- * `--dry-run`. Not a general-purpose unzip.
+ * Reader for `.uwx` containers. Handles Stored (method 0 — what our writer emits)
+ * and Deflate (method 8 — what the backend's pull `.uwx` uses); any other method
+ * throws. Not otherwise a general-purpose unzip (no Zip64, no encryption).
  *
  * @param {Buffer} buf
- * @returns {Map<string, Buffer>} name -> data
+ * @returns {Map<string, Buffer>} name -> uncompressed data
  */
 export function readZip(buf) {
   const out = new Map()
@@ -115,6 +118,7 @@ export function readZip(buf) {
     if (buf.readUInt32LE(p) !== CENTRAL_SIG) {
       throw new Error('uwx/zip: bad central directory signature')
     }
+    const method = buf.readUInt16LE(p + 10)
     const compSize = buf.readUInt32LE(p + 20)
     const nameLen = buf.readUInt16LE(p + 28)
     const extraLen = buf.readUInt16LE(p + 30)
@@ -127,7 +131,11 @@ export function readZip(buf) {
     const lNameLen = buf.readUInt16LE(localOff + 26)
     const lExtraLen = buf.readUInt16LE(localOff + 28)
     const dataStart = localOff + 30 + lNameLen + lExtraLen
-    out.set(name, buf.subarray(dataStart, dataStart + compSize))
+    const raw = buf.subarray(dataStart, dataStart + compSize)
+    // Stored (0) → verbatim; Deflate (8) → inflate the raw deflate stream.
+    if (method === 0) out.set(name, raw)
+    else if (method === 8) out.set(name, inflateRawSync(raw))
+    else throw new Error(`uwx/zip: unsupported compression method ${method} for ${name}`)
 
     p += 46 + nameLen + extraLen + commentLen
   }