@uniweb/build 0.15.5 → 0.15.6

package/src/uwx/site-project.js

@@ -100,6 +100,7 @@ const INFO_TO_SITE_YML = {
   fetcher: 'fetcher',
   build: 'build',
   search: 'search',
+  agents: 'agents',
   paths: 'paths',
   data: 'data',
   template: 'template',
@@ -259,7 +260,7 @@ export function sectionRecordToFile({ filePath, record, sourceLocale = LOCALIZED
 
 // A section/page record's durable handle: the `stable_id` content field (which
 // survives the round trip), falling back to the `$id` transport handle.
-function recordStableId(record) {
+export function recordStableId(record) {
   return record?.stable_id || record?.$id || null
 }
 
@@ -274,7 +275,7 @@ function recordStableId(record) {
 // collide on one filename (a collision would silently drop a section). The
 // `page.yml::sections:` leaf uses this same safe base so file resolution matches.
 const SAFE_STABLE_ID = /^[A-Za-z0-9_-][A-Za-z0-9._-]*$/
-function safeStableIdFilename(stableId) {
+export function safeStableIdFilename(stableId) {
   if (SAFE_STABLE_ID.test(stableId)) return stableId
   const base = stableId
     .replace(/[^A-Za-z0-9._-]+/g, '-')
@@ -361,7 +362,17 @@ function pageRecordToYml(record, sectionsArray, sourceLocale) {
   if (record.layout !== undefined) y.layout = record.layout
   if (record.seo !== undefined) y.seo = record.seo
   if (record.fetch !== undefined) y.fetch = record.fetch
-  if (sectionsArray && sectionsArray.length > 0) y.sections = sectionsArray
+  // `sections:` exists to preserve ORDER and NESTING, which the projected filenames
+  // can't carry (they're `<stableId>.md`, with no numeric prefix). It must not also
+  // decide MEMBERSHIP — and a bare list does: the collector reads a list without
+  // `...` as strict, "only listed sections processed". So a pulled page silently
+  // excluded any section added afterwards. You'd create the file, push, and be told
+  // "nothing to push", with nothing anywhere explaining why.
+  //
+  // The trailing `...` makes it inclusive: listed sections keep their order and
+  // their nesting, and anything new is discovered and appended as it would be in a
+  // page that was never pulled.
+  if (sectionsArray && sectionsArray.length > 0) y.sections = [...sectionsArray, '...']
   return y
 }
 
@@ -409,7 +420,7 @@ function projectPages(pages, pagesDir, sourceLocale, report, prune, ctx) {
 // `slug` is a localized `{lang: value}` map on the wire (a page route stays
 // localized); the directory uses the canonical SOURCE-locale slug. `param_name`
 // is already a plain string.
-function pageDirName(record, sourceLocale) {
+export function pageDirName(record, sourceLocale) {
   const slug = unwrapLocalized(record.slug, sourceLocale)
   return record.is_dynamic ? `[${record.param_name || slug}]` : slug
 }

package/src/uwx/site.js

@@ -16,10 +16,18 @@
 //
 // IDENTITY. The ENTITY `$uuid` lives in `site.yml` (top-level `$uuid`); we read it,
 // send it, and back-fill the minted value there. Nested pages/sections carry a `$id`
-// handle but NO per-item `$uuid` — the backend takes site-content content wholesale
-// (collision=force), so there is no per-item uuid round-trip on the wire. For the
-// eventual pull, per-item identity is recovered from the in-file `stableId` +
-// content-match, not a local id store.
+// handle AND a per-item `$uuid`.
+//
+// The per-item uuid is NOT authored — author files never carry sync uuids. It is
+// stamped at emit (`stampUnitUuids`) from an out-of-band cache populated by whatever
+// the backend last reported: a pull, or a push response's `finalized[].document`.
+// This is load-bearing, not bookkeeping: the backend matches records by uuid, and
+// `pages` / `page_sections` / `layout_sections` are all `multi` sections, where a
+// uuid-less record is read as NEW — inserted, with its stored counterpart deleted as
+// host-only. Sending without it replaced every page and section row on every push,
+// which silently invalidated the per-item handles the app holds for its own
+// concurrency. (This note previously said the opposite and described the wholesale
+// treatment as intended; it was neither intended nor harmless.)
 //
 // `@`-prefix child sections declared in `page.yml::nest:` ARE reconstructed —
 // they ride under their parent section's `$children` (page_sections is
@@ -238,10 +246,12 @@ const DYNAMIC_RE = /^\[(.+)\]$/
 //   - genuine self-nesting uses `$children`: a folder's child pages (within
 //     `pages`), and `@`-prefix child sections declared in `nest:` (within
 //     `page_sections`). Cross-section parentage is pure structure, never `$parent`.
-//   - identity on the wire is `$id` (the stableId — the in-file handle) at every
-//     item level, with NO per-item `$uuid` (the entity `$uuid` lives in site.yml;
-//     the backend takes site-content content wholesale). See the IDENTITY note in
-//     the file header.
+//   - `$id` (the stableId — the in-file handle) rides at every item level as the
+//     wire-only closure handle. Per-item `$uuid` is NOT authored here: it is
+//     stamped on by `stampUnitUuids` at emit, from the out-of-band identity cache,
+//     because the backend matches records by uuid and a uuid-less record in a
+//     `multi` section is read as new (inserted, stored counterpart deleted). See
+//     the IDENTITY note in the file header and `site-diff.js`.
 //
 // v0 deferrals: folder-mode `.md`-as-pages, `paths:` mounts, versioned scopes, and
 // media/asset bytes (favicon/assets, carried out of band). `@`-prefix `nest:`
@@ -253,8 +263,9 @@ const SITE_MODEL_NAME = '@uniweb/site-content'
 // `$id` (the handle), then the record's fields — the wire's canonical key order.
 // `fields` already carries `stable_id` (the Model field); `$id` is the same value.
 // Both are kept: `$id` is the sync handle, `stable_id` is the declared content field
-// the editor/render reads. No per-item `$uuid` on the wire (see the IDENTITY note in
-// the file header) — items are content of the one site-content entity.
+// the editor/render reads. Per-item `$uuid` is added later by `stampUnitUuids` (it
+// comes from the identity cache, not from the authored files) — see the IDENTITY
+// note in the file header.
 function withIdentity(id, fields) {
   return Object.assign({ $id: id }, fields)
 }
@@ -308,11 +319,21 @@ function isFullyExplicitSections(sectionsConfig) {
   )
 }
 
+// The same list plus a `...` rest marker: the named entries still give order and
+// nesting, and anything undiscovered is appended. `uniweb pull` writes exactly this
+// shape, because a bare list would make the page STRICT and silently exclude every
+// section added after the pull.
+function isExplicitWithRest(sectionsConfig) {
+  if (!Array.isArray(sectionsConfig) || sectionsConfig.length === 0) return false
+  if (!sectionsConfig.includes('...')) return false
+  return isFullyExplicitSections(sectionsConfig.filter((i) => i !== '...'))
+}
+
 // Build the page_sections tree from an explicit `sections:` array: each item is
 // a section name (string) or `{ name: [children…] }`, resolved to its file by
 // name (bare / `@` / numeric-prefix tolerant) and recursed. Order and nesting
 // come from the array, not the directory.
-async function collectPageSectionsExplicit(pageDir, siteRoot, sectionsConfig) {
+async function collectPageSectionsExplicit(pageDir, siteRoot, sectionsConfig, { appendRest = false } = {}) {
   const mdFiles = (await readdir(pageDir)).filter(isMarkdownFile).sort(compareFilenames)
   const seen = new Set()
 
@@ -335,9 +356,25 @@ async function collectPageSectionsExplicit(pageDir, siteRoot, sectionsConfig) {
 
   const sections = []
   for (const item of sectionsConfig) {
+    if (item === '...') continue // the rest marker itself names no section
     const s = await buildItem(item)
     if (s) sections.push(s)
   }
+
+  // `...` — append any top-level file the list didn't claim, in filename order, so
+  // a section added after a pull appears instead of being silently dropped.
+  // `seen` already holds every file the list used, children included, so a nested
+  // child is never also promoted to a sibling of its own parent.
+  if (appendRest) {
+    for (const file of mdFiles) {
+      if (isChildSection(file) || seen.has(file)) continue
+      seen.add(file)
+      const stableDefault = parseNumericPrefix(stripAtPrefix(parse(file).name)).name
+      const { section } = await processMarkdownFile(join(pageDir, file), String(seen.size), siteRoot, stableDefault)
+      section.subsections = []
+      sections.push(section)
+    }
+  }
   return sections.map((s, i) => sectionToRecord(s, i))
 }
 
@@ -345,6 +382,9 @@ async function collectPageSectionsNested(pageDir, siteRoot, pageConfig) {
   if (isFullyExplicitSections(pageConfig?.sections)) {
     return collectPageSectionsExplicit(pageDir, siteRoot, pageConfig.sections)
   }
+  if (isExplicitWithRest(pageConfig?.sections)) {
+    return collectPageSectionsExplicit(pageDir, siteRoot, pageConfig.sections, { appendRest: true })
+  }
   const mdFiles = (await readdir(pageDir)).filter(isMarkdownFile).sort(compareFilenames)
   const nest = pageConfig?.nest && typeof pageConfig.nest === 'object' ? pageConfig.nest : {}
 
@@ -589,6 +629,13 @@ export async function siteProjectToDocument(siteRoot, opts = {}) {
   setIf(info, 'fetcher', siteYml.fetcher)
   setIf(info, 'build', siteYml.build)
   setIf(info, 'search', siteYml.search)
+  // `agents` — the projections opt-out + route exclusions. Carried because the
+  // app is a second PUBLISHER of projections and derives them from stored
+  // content: without this block it cannot see `agents: false` or
+  // `agents.exclude`, so an author's opt-out is silently reversed and an
+  // excluded branch becomes both discoverable AND summarized by the index.
+  // (The CLI lane reads site.yml directly and honors it either way.)
+  setIf(info, 'agents', siteYml.agents)
   setIf(info, 'paths', siteYml.paths)
   setIf(info, 'data', siteYml.data ?? siteYml.fetch)
   // `app` — the deployment's `@uniweb/app-spec` reference (a bare uuid string),

package/src/uwx/sync-package.js

@@ -22,6 +22,7 @@
 import { buildCollectionEntities, entityContentHash } from './collections.js'
 import { buildFolderEntity } from './folder.js'
 import { siteProjectToDocument } from './site.js'
+import { stampUnitUuids, collectUnitUuids } from './site-diff.js'
 import { emitEntitySyncPackage } from './entity-document.js'
 import { isLocalAssetPath } from '../site/assets.js'
 
@@ -30,6 +31,33 @@ const SITE_ENTITY_KEY = 'site-content'
 
 const cacheKey = (entity) => `${entity.model} ${entity.id}`
 
+// Attach the push gate's optimistic-concurrency token to an entity, keyed by the
+// BACKEND uuid in its document body (`$uuid`). A never-synced entity has no
+// `$uuid` and therefore no base — correctly unconditional, since there is no
+// stored state it could be stale against. `baseVersions` is the caller's
+// uuid→version map (the sync-cache); an entity the map doesn't know is left
+// unconditional rather than guessed at. See entity-document.js for what the
+// token means on the wire.
+const withBaseVersion = (baseVersions, itemBaseVersions) => (entity) => {
+  const uuid = entity?.document?.$uuid
+  const v = uuid ? baseVersions[uuid] : null
+  // Per-item preconditions, narrowed to the records THIS package actually carries.
+  // The cache spans the whole site, but a package holds only what changed; sending
+  // tokens for absent records would bloat the manifest and assert preconditions on
+  // items we are not touching.
+  let items = null
+  if (itemBaseVersions && Object.keys(itemBaseVersions).length) {
+    items = {}
+    for (const itemUuid of Object.values(collectUnitUuids(entity.document))) {
+      const t = itemBaseVersions[itemUuid]
+      if (t) items[itemUuid] = t
+    }
+    if (!Object.keys(items).length) items = null
+  }
+  if (!v && !items) return entity
+  return { ...entity, ...(v ? { baseVersion: v } : {}), ...(items ? { itemBaseVersions: items } : {}) }
+}
+
 function emitLane(entities, exporter, exportedAt, extraModels = []) {
   const models = [...new Set([...entities.map((e) => e.model), ...extraModels])]
   const buffer = emitEntitySyncPackage({
@@ -113,6 +141,21 @@ function rewriteEntityAssets(node, map) {
  * @param {string} [opts.sourceLocale]    - localized-field wrap locale
  * @param {Object<string,string>} [opts.priorHashes] - sync-cache (send-only-changed)
  * @param {boolean} [opts.sendAll]        - bypass the prior-hash filter
+ * @param {Object<string,string>} [opts.itemUuids] - unit path → backend `$uuid`,
+ *        stamped onto the site-content document so the backend matches our items
+ *        instead of re-minting them (which deletes and recreates every page and
+ *        section row). Sourced from a pull or a push response, cached by the caller.
+ * @param {Object<string,string>} [opts.itemBaseVersions] - record `$uuid` → opaque
+ *        per-ITEM `version`. Narrowed at emit to the records the package carries and
+ *        sent as `entries[].item_base_versions`, so the backend can refuse only the
+ *        records that genuinely moved instead of the whole document. Omit to push
+ *        those records unconditionally.
+ * @param {Object<string,string>} [opts.baseVersions] - backend-uuid → opaque
+ *        `version` token, the push gate's optimistic-concurrency precondition.
+ *        Each sent entity whose `$uuid` the map knows carries it as a top-level
+ *        `entries[].base_version`; the backend refuses the whole package
+ *        atomically if its stored version has moved. Omit the map (or an entry)
+ *        to push unconditionally — that IS the force path.
  * @param {boolean} [opts.includeSite]    - include the site-content lane (default true)
  * @param {object} [opts.injectInfo]      - deploy-derived `info.*` to stamp on the
  *        site-content document (e.g. `{ data_bundle }`, the static-data ball URL);
@@ -140,6 +183,7 @@ export async function emitSyncPackages(siteRoot, opts = {}) {
   const sourceLocale = opts.sourceLocale
   const priorHashes = opts.priorHashes || {}
   const sendAll = !!opts.sendAll
+  const stamp = withBaseVersion(opts.baseVersions || {}, opts.itemBaseVersions || {})
   const exporter = opts.exporter
   const exportedAt = opts.exportedAt
 
@@ -166,6 +210,33 @@ export async function emitSyncPackages(siteRoot, opts = {}) {
     siteDoc.info = { ...siteDoc.info, ...opts.injectInfo }
   }
 
+  // Per-item identity. `pages`, `page_sections` and `layout_sections` are all
+  // `multi` sections on the backend's Model, and its reconcile matches a record by
+  // `$uuid`: a record without one is read as NEW, so it is inserted and the stored
+  // counterpart falls out as host-only and is DELETED. Pushing identity-blind
+  // therefore replaces every page and section row on every push — the content still
+  // lands, which is why it hid, but the app's per-item concurrency handles all point
+  // at rows that no longer exist.
+  //
+  // The uuids come from whatever the backend last told us (a pull, or a push
+  // response's `finalized[].document`), cached out-of-band by the caller so author
+  // files never carry sync uuids. A unit the map doesn't know keeps no `$uuid` —
+  // that is new content on its first push, where minting is correct.
+  //
+  // Stamping does NOT perturb the send-only-changed hash: `entityContentHash`
+  // strips `$`-sigils, so adopting this never re-fires an unchanged lane.
+  let itemIdentity = null
+  if (siteDoc && opts.itemUuids && typeof opts.itemUuids === 'object') {
+    itemIdentity = stampUnitUuids(siteDoc, opts.itemUuids, sourceLocale)
+    for (const path of itemIdentity.collisions) {
+      warnings.push(
+        `Two sections resolve to the same file (${path}) — their stable ids collide ` +
+        `(e.g. \`1-name.md\` and \`name.md\`). Only the first keeps its identity, and a pull ` +
+        `would collapse them into one file. Rename one.`
+      )
+    }
+  }
+
   // Local-media over push (Slice 5). `assetRewrite` ({ '/images/x.png': serveUrl })
   // is supplied by the deploy's SECOND emit, after it has uploaded the files the
   // FIRST emit surfaced in `localAssets`. It is absent on the collect emit and on
@@ -225,7 +296,7 @@ export async function emitSyncPackages(siteRoot, opts = {}) {
     // Folder first (always, for the `$ref` closure), then changed records. The
     // leading `{ kind: 'folder' }` keeps submission position 0 aligned for record
     // back-fill (backfillEntityUuids skips it — the folder has no uuid to write).
-    const entities = [folder, ...changedRecords.map((r) => r.entity)]
+    const entities = [folder, ...changedRecords.map((r) => r.entity)].map(stamp)
     const index = [{ kind: 'folder' }, ...changedRecords.map((r) => r.index)]
     // The folder references every record's Model via `entry.model` — including records
     // filtered out here by send-only-changed. Declare them all (the backend rejects a
@@ -237,7 +308,7 @@ export async function emitSyncPackages(siteRoot, opts = {}) {
   // --- site-content lane -------------------------------------------------------
   let siteContent = null
   if (siteEntity && changed(siteEntity)) {
-    siteContent = { ...emitLane([siteEntity], exporter, exportedAt), index: [{ kind: 'site' }] }
+    siteContent = { ...emitLane([stamp(siteEntity)], exporter, exportedAt), index: [{ kind: 'site' }] }
   }
 
   // The site's current uuid (from site.yml): the verb keys the folder push route on
@@ -245,5 +316,12 @@ export async function emitSyncPackages(siteRoot, opts = {}) {
   // the content lane by its presence/absence.
   const siteContentUuid = siteDoc?.$uuid
 
-  return { siteContent, collections, siteContentUuid, hashes, warnings, skipped, schemaless: col.schemaless, localAssets }
+  return {
+    siteContent, collections, siteContentUuid, hashes, warnings, skipped,
+    schemaless: col.schemaless, localAssets,
+    // { stamped, unknown } when identity was applied; null when the caller passed
+    // no map. `unknown > 0` with `stamped === 0` on a site that has been pushed
+    // before is the index-loss signature the backend refuses — the caller reports it.
+    itemIdentity,
+  }
 }