@uniweb/build 0.29.1 → 0.30.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (41) hide show
  1. package/package.json +5 -5
  2. package/src/content/index.js +6 -6
  3. package/src/dev-backend.js +31 -31
  4. package/src/i18n/freeform.js +44 -24
  5. package/src/i18n/index.js +22 -22
  6. package/src/i18n/{collections.js → records.js} +114 -51
  7. package/src/i18n/sync.js +9 -8
  8. package/src/site/build-site-data.js +9 -12
  9. package/src/site/config.js +1 -1
  10. package/src/site/content-collector.js +35 -40
  11. package/src/site/data-fetcher.js +23 -10
  12. package/src/site/entity-pool.js +211 -0
  13. package/src/site/fetch-shapes.js +13 -12
  14. package/src/site/foundation-ref.js +1 -1
  15. package/src/site/index.js +4 -4
  16. package/src/site/plugin.js +58 -63
  17. package/src/site/queries-config.js +324 -0
  18. package/src/site/{collection-processor.js → query-processor.js} +180 -95
  19. package/src/site/records-config.js +299 -0
  20. package/src/site/schemaless-data.js +2 -2
  21. package/src/utils/numeric-prefix.js +63 -0
  22. package/src/uwx/backfill.js +5 -5
  23. package/src/uwx/data-schema.js +2 -2
  24. package/src/uwx/entity-source.js +122 -0
  25. package/src/uwx/folder.js +92 -77
  26. package/src/uwx/index.js +33 -13
  27. package/src/uwx/locale-sync.js +2 -2
  28. package/src/uwx/project-writer.js +36 -10
  29. package/src/uwx/queries-config.js +11 -0
  30. package/src/uwx/records-project.js +535 -0
  31. package/src/uwx/{collections.js → records.js} +152 -69
  32. package/src/uwx/site-diff.js +6 -6
  33. package/src/uwx/site-project.js +30 -5
  34. package/src/uwx/site.js +295 -27
  35. package/src/uwx/sync-package.js +32 -18
  36. package/src/validate-data.js +17 -19
  37. package/src/site/collections-config.js +0 -260
  38. package/src/uwx/collection-source.js +0 -180
  39. package/src/uwx/collections-config.js +0 -9
  40. package/src/uwx/collections-project.js +0 -335
  41. /package/src/search/{collections.js → records-index.js} +0 -0
@@ -1,7 +1,7 @@
1
1
  /**
2
2
  * Collection Processor
3
3
  *
4
- * Processes content collections from markdown and YAML files into JSON data.
4
+ * Materializes each named QUERY over the site's records into JSON data.
5
5
  * Collections are defined in site.yml and processed at build time.
6
6
  *
7
7
  * ⛔ A COLLECTION `.md` IS NOT A PAGE-SECTION `.md`. Same extension, unrelated
@@ -18,7 +18,7 @@
18
18
  * ⭐ And `.md` is the HYBRID case, not the general one. It exists for records
19
19
  * that are part data and part prose — a blog article. YAML and JSON records are
20
20
  * data only, have no body, and express nesting and arrays natively; they are the
21
- * plain case rather than the exception. Reasoning about collections from the
21
+ * plain case rather than the exception. Reasoning about records from the
22
22
  * markdown shape alone imports a body and a content field that most records
23
23
  * do not have.
24
24
  *
@@ -39,15 +39,14 @@
39
39
  * @module @uniweb/build/site/collection-processor
40
40
  *
41
41
  * @example
42
- * // site.yml
43
- * collections:
44
- * articles:
45
- * path: collections/articles
46
- * sort: date desc
42
+ * // queries.yml
43
+ * articles:
44
+ * schema: '@/article'
45
+ * sort: date desc
47
46
  *
48
47
  * // Usage
49
- * const collections = await processCollections(siteDir, config.collections)
50
- * await writeCollectionFiles(siteDir, collections)
48
+ * const byQuery = await processQueries(siteDir, config.queries)
49
+ * await writeQueryFiles(siteDir, byQuery)
51
50
  */
52
51
 
53
52
  import { readFile, readdir, stat, writeFile, mkdir, copyFile, rm } from 'node:fs/promises'
@@ -56,8 +55,10 @@ import { existsSync } from 'node:fs'
56
55
  import yaml from 'js-yaml'
57
56
  import { parseBibtex } from '@citestyle/bibtex'
58
57
  import { DATA_DIR } from '@uniweb/core'
59
- import { applyFilter, applySort } from './data-fetcher.js'
58
+ import { applyWhere, applyFilter, applySort } from './data-fetcher.js'
60
59
  import { resolveAssetPath, walkContentAssets, isLocalAssetPath } from './assets.js'
60
+ import { readEntityPool, groupPoolBySchema, ENTITIES_DIR } from './entity-pool.js'
61
+ import { readRecordsConfig, resolveFolder, FOLDER_MISSING } from './records-config.js'
61
62
 
62
63
  // Try to import content-reader for markdown parsing
63
64
  let markdownToProseMirror
@@ -86,10 +87,10 @@ try {
86
87
  *
87
88
  * @example
88
89
  * // Simple form
89
- * parseCollectionConfig('articles', 'collections/articles')
90
+ * parseQueryConfig('articles', '@/article')
90
91
  *
91
92
  * // Extended form
92
- * parseCollectionConfig('articles', {
93
+ * parseQueryConfig('articles', {
93
94
  * path: 'collections/articles',
94
95
  * route: '/blog',
95
96
  * sort: 'date desc',
@@ -97,13 +98,22 @@ try {
97
98
  * limit: 100
98
99
  * })
99
100
  */
100
- function parseCollectionConfig(name, config) {
101
+ function parseQueryConfig(name, config) {
102
+ // ⚠️ `queries.yml`'s TERSEST form is a bare key — `articles:` — which YAML
103
+ // parses as NULL. The resolver normalizes that away, so the build path never
104
+ // sees it; a caller reading raw config (as this function's own docstring
105
+ // shows) would have crashed on the shortest thing an author can write.
106
+ if (config === null || config === undefined) config = {}
101
107
  if (typeof config === 'string') {
108
+ // The string shorthand names the SCHEMA — `entities/{schema}/` supplies the
109
+ // records, so there is no directory for a query to name.
102
110
  return {
103
111
  name,
104
- path: config,
112
+ schema: config,
113
+ url: null,
105
114
  route: null,
106
115
  sort: null,
116
+ where: null,
107
117
  filter: null,
108
118
  limit: 0,
109
119
  excerpt: { maxLength: 160 },
@@ -113,9 +123,16 @@ function parseCollectionConfig(name, config) {
113
123
 
114
124
  return {
115
125
  name,
116
- path: config.path,
126
+ // The query's schema selects its records from the pool — `entities/{schema}/`
127
+ // declares the model, so the entities of a schema ARE the query's records.
128
+ schema: config.schema || null,
129
+ url: config.url || null,
117
130
  route: config.route || null,
118
131
  sort: config.sort || null,
132
+ // `where:` is the CANONICAL predicate; `filter:` is the deprecated string DSL
133
+ // it replaced. Both are carried and both are applied below, in the same order
134
+ // `data-fetcher.js::applyPostProcessing` uses — see the note there.
135
+ where: config.where || null,
119
136
  filter: config.filter || null,
120
137
  limit: config.limit || 0,
121
138
  excerpt: {
@@ -131,18 +148,18 @@ function parseCollectionConfig(name, config) {
131
148
  // singular detail there) or via the kit's useEntityDetail hook.
132
149
  deferred: Array.isArray(config.deferred) ? config.deferred.slice() : null,
133
150
  // `detailUrl:` names the per-record endpoint pattern for API-backed
134
- // collections (where the build emits no per-record files because
151
+ // remote sources (where the build emits no per-record files because
135
152
  // there are no on-disk source files to materialize). Used by the
136
153
  // runtime's auto-detail injection and the useEntityDetail kit hook.
137
154
  // Pattern uses {slug} as the placeholder; substitution at runtime
138
155
  // pulls from the dynamic-route param (entity-store) or the record's
139
- // slug field (useEntityDetail). Markdown-backed collections leave
156
+ // slug field (useEntityDetail). File-backed queries leave
140
157
  // this null and get the static-file default /data/<name>/<slug>.json.
141
158
  detailUrl: typeof config.detailUrl === 'string' ? config.detailUrl : null,
142
159
  // `queryable:` declares the queryable surface — which fields a
143
160
  // foundation can offer for filtering UI, with their type and
144
161
  // type-specific metadata (enum options, range bounds). Foundations
145
- // read this metadata via the kit's useCollectionQueryable hook to
162
+ // read this metadata via the kit's useQueryable hook to
146
163
  // render filter controls and compose where-objects from user
147
164
  // interactions. The framework doesn't validate the shape here —
148
165
  // foundations get whatever the author wrote; documentation defines
@@ -170,7 +187,7 @@ function parseCollectionConfig(name, config) {
170
187
  * fields and moved the page from /blog/docs-sites to /blog/11_docs_sites. The
171
188
  * only trace was
172
189
  *
173
- * [collection-processor] YAML parse error: bad indentation of a mapping entry (4:72)
190
+ * [query-processor] YAML parse error: bad indentation of a mapping entry (4:72)
174
191
  *
175
192
  * on line 16 of 857 lines of build output, naming no file, nine lines above
176
193
  * "Processed articles: 6 items" — a success line that reads as everything
@@ -303,20 +320,26 @@ function isExternalUrl(src) {
303
320
  /**
304
321
  * Process assets in collection content
305
322
  * - Resolves relative paths to site-root-relative paths
306
- * - Copies co-located assets to public/collections/<collection>/
323
+ * - Copies co-located assets to public/records/<schema>/
307
324
  * - Updates paths in the content in place
308
325
  *
309
326
  * @param {Object} content - ProseMirror document
310
327
  * @param {string} itemPath - Path to the markdown file
311
328
  * @param {string} siteRoot - Site root directory
312
- * @param {string} collectionName - Name of the collection (e.g., 'articles')
329
+ * @param {string} queryName - Name of the collection (e.g., 'articles')
313
330
  * @returns {Promise<Object>} Asset manifest for this item
314
331
  */
315
- async function processCollectionAssets(content, itemPath, siteRoot, collectionName, basePath) {
332
+ async function processRecordAssets(content, itemPath, siteRoot, poolDirs, basePath) {
316
333
  const assets = {}
317
334
  const itemDir = dirname(itemPath)
318
335
  const publicDir = join(siteRoot, 'public')
319
- const targetDir = join(publicDir, 'collections', collectionName)
336
+ // THE RECORD'S OWN HOME, keyed by its pool position — `public/records/<schema
337
+ // dirs>/`. It was `public/collections/<queryName>/`, which meant the SAME image
338
+ // was copied once per query that returned the record, under two URLs. Third
339
+ // instance of the same conflation (after the freeform locale tree and the
340
+ // translation manifest): an asset belongs to a record, and which query selects
341
+ // it is not a fact about it.
342
+ const targetDir = join(publicDir, 'records', poolDirs)
320
343
 
321
344
  // Walk content and collect asset paths
322
345
  const assetNodes = []
@@ -349,7 +372,7 @@ async function processCollectionAssets(content, itemPath, siteRoot, collectionNa
349
372
  await copyFile(result.resolved, targetPath)
350
373
 
351
374
  // Update path to site-root-relative
352
- finalPath = `${basePath}collections/${collectionName}/${assetFilename}`
375
+ finalPath = `${basePath}records/${poolDirs}/${assetFilename}`
353
376
 
354
377
  assets[src] = {
355
378
  original: src,
@@ -384,7 +407,7 @@ async function processCollectionAssets(content, itemPath, siteRoot, collectionNa
384
407
  const posterTarget = join(targetDir, posterFilename)
385
408
  await mkdir(targetDir, { recursive: true })
386
409
  await copyFile(posterResult.resolved, posterTarget)
387
- node.attrs.poster = `${basePath}collections/${collectionName}/${posterFilename}`
410
+ node.attrs.poster = `${basePath}records/${poolDirs}/${posterFilename}`
388
411
  }
389
412
  }
390
413
 
@@ -395,7 +418,7 @@ async function processCollectionAssets(content, itemPath, siteRoot, collectionNa
395
418
  const previewTarget = join(targetDir, previewFilename)
396
419
  await mkdir(targetDir, { recursive: true })
397
420
  await copyFile(previewResult.resolved, previewTarget)
398
- node.attrs.preview = `${basePath}collections/${collectionName}/${previewFilename}`
421
+ node.attrs.preview = `${basePath}records/${poolDirs}/${previewFilename}`
399
422
  }
400
423
  }
401
424
  }
@@ -406,17 +429,17 @@ async function processCollectionAssets(content, itemPath, siteRoot, collectionNa
406
429
  /**
407
430
  * Process assets in a data item (YAML/JSON)
408
431
  * - Recursively walks the data object looking for local asset paths
409
- * - Copies co-located assets to public/collections/<collection>/
432
+ * - Copies co-located assets to public/records/<schema>/
410
433
  * - Rewrites paths to absolute URLs (with base path)
411
434
  *
412
435
  * @param {Object} data - Parsed data object (mutated in place)
413
436
  * @param {string} itemPath - Path to the data file
414
437
  * @param {string} siteRoot - Site root directory
415
- * @param {string} collectionName - Name of the collection
438
+ * @param {string} queryName - Name of the collection
416
439
  * @param {string} basePath - Site base path (e.g., '/' or '/docs/')
417
440
  */
418
- async function processDataItemAssets(data, itemPath, siteRoot, collectionName, basePath) {
419
- const targetDir = join(siteRoot, 'public', 'collections', collectionName)
441
+ async function processDataItemAssets(data, itemPath, siteRoot, poolDirs, basePath) {
442
+ const targetDir = join(siteRoot, 'public', 'records', poolDirs)
420
443
 
421
444
  async function walk(parent, key) {
422
445
  const val = parent[key]
@@ -427,7 +450,7 @@ async function processDataItemAssets(data, itemPath, siteRoot, collectionName, b
427
450
  const filename = basename(resolved)
428
451
  await mkdir(targetDir, { recursive: true })
429
452
  await copyFile(resolved, join(targetDir, filename))
430
- parent[key] = `${basePath}collections/${collectionName}/${filename}`
453
+ parent[key] = `${basePath}records/${poolDirs}/${filename}`
431
454
  }
432
455
  } else if (val.startsWith('/')) {
433
456
  // Absolute site path — just prepend base
@@ -467,7 +490,7 @@ async function processDataItemAssets(data, itemPath, siteRoot, collectionName, b
467
490
  * @param {string} filename - YAML filename (.yml or .yaml)
468
491
  * @returns {Promise<Object|Array|null>} Processed item(s) or null if unpublished
469
492
  */
470
- async function processDataItem(dir, filename, siteRoot, collectionName, basePath) {
493
+ async function processDataItem(dir, filename, siteRoot, poolDirs, basePath) {
471
494
  const filepath = join(dir, filename)
472
495
  const raw = await readFile(filepath, 'utf-8')
473
496
  const data = yaml.load(raw) || {}
@@ -476,7 +499,7 @@ async function processDataItem(dir, filename, siteRoot, collectionName, basePath
476
499
  if (Array.isArray(data)) {
477
500
  for (const item of data) {
478
501
  if (item && typeof item === 'object') {
479
- await processDataItemAssets(item, filepath, siteRoot, collectionName, basePath)
502
+ await processDataItemAssets(item, filepath, siteRoot, poolDirs, basePath)
480
503
  }
481
504
  }
482
505
  return data
@@ -486,7 +509,7 @@ async function processDataItem(dir, filename, siteRoot, collectionName, basePath
486
509
  if (data.published === false) return null
487
510
  const slug = basename(filename, extname(filename))
488
511
  const item = { slug, ...data }
489
- await processDataItemAssets(item, filepath, siteRoot, collectionName, basePath)
512
+ await processDataItemAssets(item, filepath, siteRoot, poolDirs, basePath)
490
513
  return item
491
514
  }
492
515
 
@@ -501,7 +524,7 @@ async function processDataItem(dir, filename, siteRoot, collectionName, basePath
501
524
  * @param {string} filename - JSON filename
502
525
  * @returns {Promise<Object|Array|null>} Processed item(s) or null if unpublished
503
526
  */
504
- async function processJsonItem(dir, filename, siteRoot, collectionName, basePath) {
527
+ async function processJsonItem(dir, filename, siteRoot, poolDirs, basePath) {
505
528
  const filepath = join(dir, filename)
506
529
  const raw = await readFile(filepath, 'utf-8')
507
530
  const slug = basename(filename, '.json')
@@ -511,7 +534,7 @@ async function processJsonItem(dir, filename, siteRoot, collectionName, basePath
511
534
  if (Array.isArray(data)) {
512
535
  for (const item of data) {
513
536
  if (item && typeof item === 'object') {
514
- await processDataItemAssets(item, filepath, siteRoot, collectionName, basePath)
537
+ await processDataItemAssets(item, filepath, siteRoot, poolDirs, basePath)
515
538
  }
516
539
  }
517
540
  return data
@@ -520,7 +543,7 @@ async function processJsonItem(dir, filename, siteRoot, collectionName, basePath
520
543
  // Object → single item
521
544
  if (data.published === false) return null
522
545
  const item = { slug, ...data }
523
- await processDataItemAssets(item, filepath, siteRoot, collectionName, basePath)
546
+ await processDataItemAssets(item, filepath, siteRoot, poolDirs, basePath)
524
547
  return item
525
548
  }
526
549
 
@@ -556,7 +579,7 @@ async function processBibtexItem(dir, filename) {
556
579
  * @param {string} siteRoot - Site root directory for asset resolution
557
580
  * @returns {Promise<Object|null>} Processed item or null if unpublished
558
581
  */
559
- async function processContentItem(dir, filename, config, siteRoot, basePath) {
582
+ async function processContentItem(dir, filename, config, siteRoot, basePath, poolDirs) {
560
583
  const filepath = join(dir, filename)
561
584
  const raw = await readFile(filepath, 'utf-8')
562
585
  const slug = basename(filename, extname(filename))
@@ -574,13 +597,13 @@ async function processContentItem(dir, filename, config, siteRoot, basePath) {
574
597
 
575
598
  // Process assets (resolve paths, copy co-located files)
576
599
  // This modifies content in place, updating paths to site-root-relative
577
- await processCollectionAssets(content, filepath, siteRoot, config.name, basePath)
600
+ await processRecordAssets(content, filepath, siteRoot, poolDirs, basePath)
578
601
 
579
602
  // Extract excerpt
580
603
  const excerpt = extractExcerpt(frontmatter, content, config.excerpt)
581
604
 
582
605
  // Extract first image (frontmatter takes precedence)
583
- // Note: paths in content have already been updated by processCollectionAssets
606
+ // Note: paths in content have already been updated by processRecordAssets
584
607
  const image = frontmatter.image || extractFirstImage(content)
585
608
 
586
609
  return {
@@ -631,7 +654,7 @@ async function collectSourceFiles(dir, rel = '') {
631
654
  * whichever sorts last wins the route and the per-record file. That is a real
632
655
  * ambiguity only the author can resolve, so it is reported rather than repaired.
633
656
  */
634
- function warnDuplicateSlugs(items, collectionName) {
657
+ function warnDuplicateSlugs(items, queryName) {
635
658
  const seen = new Map()
636
659
  for (const item of items) {
637
660
  if (!item || item.slug === undefined) continue
@@ -639,7 +662,7 @@ function warnDuplicateSlugs(items, collectionName) {
639
662
  const where = item.path ? `${item.path}/` : ''
640
663
  if (seen.has(slug)) {
641
664
  console.warn(
642
- `[collection-processor] Collection "${collectionName}" has more than one record with ` +
665
+ `[query-processor] Query "${queryName}" has more than one record with ` +
643
666
  `slug "${slug}" (${seen.get(slug)}${slug}, ${where}${slug}). Its detail route and ` +
644
667
  `per-record file resolve to only one of them — give them distinct slugs.`
645
668
  )
@@ -656,41 +679,49 @@ function warnDuplicateSlugs(items, collectionName) {
656
679
  * @param {Object} config - Parsed collection config
657
680
  * @returns {Promise<Array>} Array of processed items
658
681
  */
659
- async function collectItems(siteDir, config, collectionsBase, basePath) {
660
- const base = collectionsBase || siteDir
661
- const collectionDir = resolve(base, config.path)
662
-
663
- // Check if collection directory exists
664
- if (!existsSync(collectionDir)) {
665
- console.warn(`[collection-processor] Collection folder not found: ${config.path}`)
666
- return []
667
- }
668
-
669
- const itemFiles = await collectSourceFiles(collectionDir)
670
-
671
- // Process all collection files (markdown → content items, YAML/JSON → data
672
- // items, BibTeX → CSL-JSON bibliography items).
682
+ async function collectItems(siteDir, config, entitiesDir, basePath) {
683
+ // THE QUERY NAMES A SCHEMA AND THE POOL FOLLOWS — the same resolution the
684
+ // sync lane makes, from the same reader, so the two lanes cannot disagree
685
+ // about which files are a query's records. They used to: this one recursed
686
+ // into a collection directory and sync did not.
687
+ const pooled = config.poolEntities || []
688
+ if (pooled.length === 0) return []
689
+
690
+ const dirOf = (e) => resolve(siteDir, entitiesDir || ENTITIES_DIR, ...e.dirs)
691
+
692
+ // Process all entity files (markdown → content items, YAML/JSON → data items,
693
+ // BibTeX → CSL-JSON bibliography items).
673
694
  let items = await Promise.all(
674
- itemFiles.map(file => {
675
- if (file.endsWith('.bib')) {
676
- return processBibtexItem(collectionDir, file)
695
+ pooled.map((e) => {
696
+ const dir = dirOf(e)
697
+ const file = `${e.slug}${e.ext}`
698
+ if (e.ext === '.bib') {
699
+ return processBibtexItem(dir, file)
677
700
  }
678
- if (file.endsWith('.json')) {
679
- return processJsonItem(collectionDir, file, siteDir, config.name, basePath)
701
+ if (e.ext === '.json') {
702
+ return processJsonItem(dir, file, siteDir, e.dirs.join('/'), basePath)
680
703
  }
681
- if (file.endsWith('.yml') || file.endsWith('.yaml')) {
682
- return processDataItem(collectionDir, file, siteDir, config.name, basePath)
704
+ if (e.ext === '.yml' || e.ext === '.yaml') {
705
+ return processDataItem(dir, file, siteDir, e.dirs.join('/'), basePath)
683
706
  }
684
- return processContentItem(collectionDir, file, config, siteDir, basePath)
707
+ return processContentItem(dir, file, config, siteDir, basePath, e.dirs.join('/'))
685
708
  })
686
709
  )
687
710
 
688
- // Stamp each record's position inside the collection BEFORE flattening, while
689
- // a result is still aligned with the file it came from. A file's own array
690
- // entries (array-form YAML/JSON, every .bib entry) all share its directory.
711
+ // `path` IS THE PLACEMENT `records.yml` GAVE THE RECORD, and it is the whole
712
+ // reason folders exist: `where: { path: { under: 'archive' } }` is how a query
713
+ // asks for a slice. Structure is query scope, not navigation.
714
+ //
715
+ // ⛔ THIS WAS HARDCODED TO `''` FOR A WHILE, AND THE COMMENT SAID "until the
716
+ // folder producer lands". It landed, and this was not revisited — so every
717
+ // folder slice matched NOTHING on the delivery lane, silently, which is the one
718
+ // failure mode the whole design is built to prevent. Measured before the fix: a
719
+ // two-record site with an `archive` folder returned `[]` for its own slice.
720
+ //
721
+ // ⚠️ It stays a SCALAR. `matchUnder` in `core/src/where.js` is string-only, so
722
+ // an array would match nothing — one placement per entity is the ruling.
691
723
  items = items.map((result, i) => {
692
- const dir = dirname(itemFiles[i])
693
- const path = dir === '.' ? '' : dir
724
+ const path = pooled[i] ? (config.placements?.get(pooled[i].id)?.path ?? '') : ''
694
725
  if (Array.isArray(result)) return result.map((item) => item && { ...item, path })
695
726
  return result && { ...result, path }
696
727
  })
@@ -713,7 +744,21 @@ async function collectItems(siteDir, config, collectionsBase, basePath) {
713
744
  }))
714
745
  }
715
746
 
716
- // Apply custom filter
747
+ // ORDER MATCHES `data-fetcher.js::applyPostProcessing` — where, filter, sort,
748
+ // limit. Two lanes evaluate the same declaration (this one materializes a query
749
+ // to `/data/<name>.json`; that one runs a page-level `fetch:`), so a difference
750
+ // in order is a difference in RESULT for any query that both narrows and limits.
751
+ //
752
+ // ⚠️ `where` was missing here entirely until 2026-08-29: `parseQueryConfig`
753
+ // read `filter` and never `where`, so the CANONICAL predicate was parsed, put on
754
+ // the sync wire, stored — and never applied, while the DEPRECATED one it replaced
755
+ // worked. An author following current guidance got silence and shipped unfiltered
756
+ // data. Pinned by `tests/collection-query-terms.test.js`.
757
+ if (config.where) {
758
+ items = applyWhere(items, config.where)
759
+ }
760
+
761
+ // Apply the legacy filter expression (deprecated)
717
762
  if (config.filter) {
718
763
  items = applyFilter(items, config.filter)
719
764
  }
@@ -735,28 +780,68 @@ async function collectItems(siteDir, config, collectionsBase, basePath) {
735
780
  * Process all content collections defined in site.yml
736
781
  *
737
782
  * @param {string} siteDir - Site root directory
738
- * @param {Object} collectionsConfig - Collections config from site.yml
783
+ * @param {Object} queriesConfig - the resolved QUERY declarations
784
+ * @param {string} [entitiesDir] - pool directory override (`site.yml::paths.entities`)
739
785
  * @returns {Promise<Object>} Map of collection name to items array
740
786
  *
741
787
  * @example
742
- * const collections = await processCollections('/path/to/site', {
788
+ * const collections = await processQueries('/path/to/site', {
743
789
  * articles: { path: 'collections/articles', sort: 'date desc' },
744
790
  * products: 'collections/products'
745
791
  * })
746
792
  * // { articles: [...], products: [...] }
747
793
  */
748
- export async function processCollections(siteDir, collectionsConfig, collectionsBase, basePath = '/') {
749
- if (!collectionsConfig || typeof collectionsConfig !== 'object') {
794
+ export async function processQueries(siteDir, queriesConfig, entitiesDir, basePath = '/') {
795
+ if (!queriesConfig || typeof queriesConfig !== 'object') {
750
796
  return {}
751
797
  }
752
798
 
799
+ // ⭐ ONE POOL WALK FOR EVERY QUERY. Two queries over the same schema read one
800
+ // set of files; a query reads none of another schema's.
801
+ const pool = await readEntityPool(siteDir, { dir: entitiesDir })
802
+ if (pool.errors.length) {
803
+ for (const e of pool.errors) console.warn(`[query-processor] ${e}`)
804
+ }
805
+
806
+ // ⛔ `records.yml` DECIDES WHAT IS PUBLISHED ON THIS LANE TOO, and it did not
807
+ // until now. Only the sync lane honoured it, so removing a record from
808
+ // `records.yml` left it shipping in `/data/<name>.json` on every static host —
809
+ // an author unpublishes a draft and it stays public. Measured before the fix.
810
+ //
811
+ // ⚖️ MISSING IS NOT EMPTY HERE EITHER, but it means something different from
812
+ // what it means to sync. There is no server folder to leave alone, so a site
813
+ // with no `records.yml` is simply not managing publication, and its whole pool
814
+ // is delivered. (Making missing mean "publish nothing" would turn every site
815
+ // without the file into a silently empty one.)
816
+ const recordsCfg = await readRecordsConfig(siteDir)
817
+ if (recordsCfg.error) console.warn(`[query-processor] ${recordsCfg.error}`)
818
+ const managed = recordsCfg.state !== FOLDER_MISSING
819
+ const folder = managed ? resolveFolder(recordsCfg.entries, pool.entities) : null
820
+ if (folder) {
821
+ for (const e of folder.errors) console.error(`[query-processor] ${e}`)
822
+ }
823
+ const published = folder
824
+ ? pool.entities.filter((e) => folder.placements.has(e.id))
825
+ : pool.entities
826
+
827
+ const poolBySchema = groupPoolBySchema(published)
828
+
753
829
  const results = {}
754
830
 
755
- for (const [name, config] of Object.entries(collectionsConfig)) {
756
- const parsed = parseCollectionConfig(name, config)
757
- const items = await collectItems(siteDir, parsed, collectionsBase, basePath)
831
+ for (const [name, config] of Object.entries(queriesConfig)) {
832
+ const parsed = parseQueryConfig(name, config)
833
+ parsed.poolEntities = parsed.schema ? poolBySchema.get(parsed.schema) || [] : []
834
+ parsed.placements = folder?.placements ?? null
835
+ if (parsed.poolEntities.length === 0 && !parsed.url) {
836
+ console.warn(
837
+ `[query-processor] Query "${name}" matches no records — nothing ` +
838
+ `published declares ${parsed.schema || '(no schema)'}. ` +
839
+ (managed ? 'Check records.yml lists them.' : 'Check entities/.')
840
+ )
841
+ }
842
+ const items = await collectItems(siteDir, parsed, entitiesDir, basePath)
758
843
  results[name] = items
759
- console.log(`[collection-processor] Processed ${name}: ${items.length} items`)
844
+ console.log(`[query-processor] Processed ${name}: ${items.length} items`)
760
845
  }
761
846
 
762
847
  return results
@@ -802,7 +887,7 @@ async function pruneOrphanedRecords(dataDir, name, expected) {
802
887
  const contained = resolve(recordsDir)
803
888
  if (contained !== resolve(dataDir, name) || !contained.startsWith(resolve(dataDir) + sep)) {
804
889
  console.warn(
805
- `[collection-processor] Refusing to prune "${name}" — it does not resolve ` +
890
+ `[query-processor] Refusing to prune "${name}" — it does not resolve ` +
806
891
  `inside ${dataDir}`
807
892
  )
808
893
  return []
@@ -826,22 +911,22 @@ async function pruneOrphanedRecords(dataDir, name, expected) {
826
911
  * @returns {Promise<void>}
827
912
  *
828
913
  * @example
829
- * await writeCollectionFiles('/path/to/site', {
914
+ * await writeQueryFiles('/path/to/site', {
830
915
  * articles: [{ slug: 'hello', title: 'Hello World', ... }]
831
916
  * })
832
917
  * // Creates public/data/articles.json
833
918
  */
834
- export async function writeCollectionFiles(siteDir, collections, collectionsConfig = null) {
835
- if (!collections || Object.keys(collections).length === 0) {
919
+ export async function writeQueryFiles(siteDir, byQuery, queriesConfig = null) {
920
+ if (!byQuery || Object.keys(byQuery).length === 0) {
836
921
  return
837
922
  }
838
923
 
839
924
  const dataDir = join(siteDir, 'public', DATA_DIR)
840
925
  await mkdir(dataDir, { recursive: true })
841
926
 
842
- for (const [name, items] of Object.entries(collections)) {
843
- const rawConfig = collectionsConfig?.[name]
844
- const parsed = rawConfig ? parseCollectionConfig(name, rawConfig) : null
927
+ for (const [name, items] of Object.entries(byQuery)) {
928
+ const rawConfig = queriesConfig?.[name]
929
+ const parsed = rawConfig ? parseQueryConfig(name, rawConfig) : null
845
930
  const deferred = parsed?.deferred
846
931
 
847
932
  if (deferred && deferred.length > 0) {
@@ -872,21 +957,21 @@ export async function writeCollectionFiles(siteDir, collections, collectionsConf
872
957
  const cascadePath = join(dataDir, `${name}.json`)
873
958
  await writeFile(cascadePath, JSON.stringify(stripped, null, 2))
874
959
  console.log(
875
- `[collection-processor] Generated ${cascadePath} (${items.length} items, ` +
960
+ `[query-processor] Generated ${cascadePath} (${items.length} items, ` +
876
961
  `deferred: [${deferred.join(', ')}]) + ${perRecordCount} per-record files`
877
962
  )
878
963
  if (pruned.length > 0) {
879
964
  // A deletion is always worth naming. These files were public a moment
880
965
  // ago, so "which ones went" is the question an author will have.
881
966
  console.log(
882
- `[collection-processor] Removed ${pruned.length} stale per-record ` +
967
+ `[query-processor] Removed ${pruned.length} stale per-record ` +
883
968
  `file(s) from ${recordsDir}: ${pruned.join(', ')}`
884
969
  )
885
970
  }
886
971
  } else {
887
972
  const filepath = join(dataDir, `${name}.json`)
888
973
  await writeFile(filepath, JSON.stringify(items, null, 2))
889
- console.log(`[collection-processor] Generated ${filepath} (${items.length} items)`)
974
+ console.log(`[query-processor] Generated ${filepath} (${items.length} items)`)
890
975
 
891
976
  // This collection is not deferred, so it has no per-record files. If it
892
977
  // used to, the directory is still there and will never be written again
@@ -894,7 +979,7 @@ export async function writeCollectionFiles(siteDir, collections, collectionsConf
894
979
  const pruned = await pruneOrphanedRecords(dataDir, name, new Set())
895
980
  if (pruned.length > 0) {
896
981
  console.log(
897
- `[collection-processor] Removed ${pruned.length} per-record file(s) ` +
982
+ `[query-processor] Removed ${pruned.length} per-record file(s) ` +
898
983
  `from ${join(dataDir, name)} — "${name}" no longer declares deferred:`
899
984
  )
900
985
  }
@@ -909,15 +994,15 @@ export async function writeCollectionFiles(siteDir, collections, collectionsConf
909
994
  * @param {Object} config - Collection config
910
995
  * @returns {Promise<Date|null>} Most recent modification time
911
996
  */
912
- export async function getCollectionLastModified(siteDir, config) {
913
- const parsed = parseCollectionConfig('temp', config)
914
- const collectionDir = join(siteDir, parsed.path)
997
+ export async function getQueryLastModified(siteDir, config) {
998
+ const parsed = parseQueryConfig('temp', config)
999
+ const poolDir = join(siteDir, parsed.path)
915
1000
 
916
- if (!existsSync(collectionDir)) {
1001
+ if (!existsSync(poolDir)) {
917
1002
  return null
918
1003
  }
919
1004
 
920
- const files = await readdir(collectionDir)
1005
+ const files = await readdir(poolDir)
921
1006
  const itemFiles = files.filter(f =>
922
1007
  !f.startsWith('_') &&
923
1008
  (f.endsWith('.md') || f.endsWith('.yml') || f.endsWith('.yaml') || f.endsWith('.json') || f.endsWith('.bib'))
@@ -926,7 +1011,7 @@ export async function getCollectionLastModified(siteDir, config) {
926
1011
  let lastModified = null
927
1012
 
928
1013
  for (const file of itemFiles) {
929
- const fileStat = await stat(join(collectionDir, file))
1014
+ const fileStat = await stat(join(poolDir, file))
930
1015
  if (!lastModified || fileStat.mtime > lastModified) {
931
1016
  lastModified = fileStat.mtime
932
1017
  }