@uniweb/build 0.29.1 → 0.30.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/package.json +5 -5
- package/src/content/index.js +6 -6
- package/src/dev-backend.js +31 -31
- package/src/i18n/freeform.js +44 -24
- package/src/i18n/index.js +22 -22
- package/src/i18n/{collections.js → records.js} +114 -51
- package/src/i18n/sync.js +9 -8
- package/src/site/build-site-data.js +9 -12
- package/src/site/config.js +1 -1
- package/src/site/content-collector.js +35 -40
- package/src/site/data-fetcher.js +23 -10
- package/src/site/entity-pool.js +211 -0
- package/src/site/fetch-shapes.js +13 -12
- package/src/site/foundation-ref.js +1 -1
- package/src/site/index.js +4 -4
- package/src/site/plugin.js +58 -63
- package/src/site/queries-config.js +324 -0
- package/src/site/{collection-processor.js → query-processor.js} +180 -95
- package/src/site/records-config.js +299 -0
- package/src/site/schemaless-data.js +2 -2
- package/src/utils/numeric-prefix.js +63 -0
- package/src/uwx/backfill.js +5 -5
- package/src/uwx/data-schema.js +2 -2
- package/src/uwx/entity-source.js +122 -0
- package/src/uwx/folder.js +92 -77
- package/src/uwx/index.js +33 -13
- package/src/uwx/locale-sync.js +2 -2
- package/src/uwx/project-writer.js +36 -10
- package/src/uwx/queries-config.js +11 -0
- package/src/uwx/records-project.js +535 -0
- package/src/uwx/{collections.js → records.js} +152 -69
- package/src/uwx/site-diff.js +6 -6
- package/src/uwx/site-project.js +30 -5
- package/src/uwx/site.js +295 -27
- package/src/uwx/sync-package.js +32 -18
- package/src/validate-data.js +17 -19
- package/src/site/collections-config.js +0 -260
- package/src/uwx/collection-source.js +0 -180
- package/src/uwx/collections-config.js +0 -9
- package/src/uwx/collections-project.js +0 -335
- /package/src/search/{collections.js → records-index.js} +0 -0
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
/**
|
|
2
2
|
* Collection Processor
|
|
3
3
|
*
|
|
4
|
-
*
|
|
4
|
+
* Materializes each named QUERY over the site's records into JSON data.
|
|
5
5
|
* Collections are defined in site.yml and processed at build time.
|
|
6
6
|
*
|
|
7
7
|
* ⛔ A COLLECTION `.md` IS NOT A PAGE-SECTION `.md`. Same extension, unrelated
|
|
@@ -18,7 +18,7 @@
|
|
|
18
18
|
* ⭐ And `.md` is the HYBRID case, not the general one. It exists for records
|
|
19
19
|
* that are part data and part prose — a blog article. YAML and JSON records are
|
|
20
20
|
* data only, have no body, and express nesting and arrays natively; they are the
|
|
21
|
-
* plain case rather than the exception. Reasoning about
|
|
21
|
+
* plain case rather than the exception. Reasoning about records from the
|
|
22
22
|
* markdown shape alone imports a body and a content field that most records
|
|
23
23
|
* do not have.
|
|
24
24
|
*
|
|
@@ -39,15 +39,14 @@
|
|
|
39
39
|
* @module @uniweb/build/site/collection-processor
|
|
40
40
|
*
|
|
41
41
|
* @example
|
|
42
|
-
* //
|
|
43
|
-
*
|
|
44
|
-
*
|
|
45
|
-
*
|
|
46
|
-
* sort: date desc
|
|
42
|
+
* // queries.yml
|
|
43
|
+
* articles:
|
|
44
|
+
* schema: '@/article'
|
|
45
|
+
* sort: date desc
|
|
47
46
|
*
|
|
48
47
|
* // Usage
|
|
49
|
-
* const
|
|
50
|
-
* await
|
|
48
|
+
* const byQuery = await processQueries(siteDir, config.queries)
|
|
49
|
+
* await writeQueryFiles(siteDir, byQuery)
|
|
51
50
|
*/
|
|
52
51
|
|
|
53
52
|
import { readFile, readdir, stat, writeFile, mkdir, copyFile, rm } from 'node:fs/promises'
|
|
@@ -56,8 +55,10 @@ import { existsSync } from 'node:fs'
|
|
|
56
55
|
import yaml from 'js-yaml'
|
|
57
56
|
import { parseBibtex } from '@citestyle/bibtex'
|
|
58
57
|
import { DATA_DIR } from '@uniweb/core'
|
|
59
|
-
import { applyFilter, applySort } from './data-fetcher.js'
|
|
58
|
+
import { applyWhere, applyFilter, applySort } from './data-fetcher.js'
|
|
60
59
|
import { resolveAssetPath, walkContentAssets, isLocalAssetPath } from './assets.js'
|
|
60
|
+
import { readEntityPool, groupPoolBySchema, ENTITIES_DIR } from './entity-pool.js'
|
|
61
|
+
import { readRecordsConfig, resolveFolder, FOLDER_MISSING } from './records-config.js'
|
|
61
62
|
|
|
62
63
|
// Try to import content-reader for markdown parsing
|
|
63
64
|
let markdownToProseMirror
|
|
@@ -86,10 +87,10 @@ try {
|
|
|
86
87
|
*
|
|
87
88
|
* @example
|
|
88
89
|
* // Simple form
|
|
89
|
-
*
|
|
90
|
+
* parseQueryConfig('articles', '@/article')
|
|
90
91
|
*
|
|
91
92
|
* // Extended form
|
|
92
|
-
*
|
|
93
|
+
* parseQueryConfig('articles', {
|
|
93
94
|
* path: 'collections/articles',
|
|
94
95
|
* route: '/blog',
|
|
95
96
|
* sort: 'date desc',
|
|
@@ -97,13 +98,22 @@ try {
|
|
|
97
98
|
* limit: 100
|
|
98
99
|
* })
|
|
99
100
|
*/
|
|
100
|
-
function
|
|
101
|
+
function parseQueryConfig(name, config) {
|
|
102
|
+
// ⚠️ `queries.yml`'s TERSEST form is a bare key — `articles:` — which YAML
|
|
103
|
+
// parses as NULL. The resolver normalizes that away, so the build path never
|
|
104
|
+
// sees it; a caller reading raw config (as this function's own docstring
|
|
105
|
+
// shows) would have crashed on the shortest thing an author can write.
|
|
106
|
+
if (config === null || config === undefined) config = {}
|
|
101
107
|
if (typeof config === 'string') {
|
|
108
|
+
// The string shorthand names the SCHEMA — `entities/{schema}/` supplies the
|
|
109
|
+
// records, so there is no directory for a query to name.
|
|
102
110
|
return {
|
|
103
111
|
name,
|
|
104
|
-
|
|
112
|
+
schema: config,
|
|
113
|
+
url: null,
|
|
105
114
|
route: null,
|
|
106
115
|
sort: null,
|
|
116
|
+
where: null,
|
|
107
117
|
filter: null,
|
|
108
118
|
limit: 0,
|
|
109
119
|
excerpt: { maxLength: 160 },
|
|
@@ -113,9 +123,16 @@ function parseCollectionConfig(name, config) {
|
|
|
113
123
|
|
|
114
124
|
return {
|
|
115
125
|
name,
|
|
116
|
-
|
|
126
|
+
// The query's schema selects its records from the pool — `entities/{schema}/`
|
|
127
|
+
// declares the model, so the entities of a schema ARE the query's records.
|
|
128
|
+
schema: config.schema || null,
|
|
129
|
+
url: config.url || null,
|
|
117
130
|
route: config.route || null,
|
|
118
131
|
sort: config.sort || null,
|
|
132
|
+
// `where:` is the CANONICAL predicate; `filter:` is the deprecated string DSL
|
|
133
|
+
// it replaced. Both are carried and both are applied below, in the same order
|
|
134
|
+
// `data-fetcher.js::applyPostProcessing` uses — see the note there.
|
|
135
|
+
where: config.where || null,
|
|
119
136
|
filter: config.filter || null,
|
|
120
137
|
limit: config.limit || 0,
|
|
121
138
|
excerpt: {
|
|
@@ -131,18 +148,18 @@ function parseCollectionConfig(name, config) {
|
|
|
131
148
|
// singular detail there) or via the kit's useEntityDetail hook.
|
|
132
149
|
deferred: Array.isArray(config.deferred) ? config.deferred.slice() : null,
|
|
133
150
|
// `detailUrl:` names the per-record endpoint pattern for API-backed
|
|
134
|
-
//
|
|
151
|
+
// remote sources (where the build emits no per-record files because
|
|
135
152
|
// there are no on-disk source files to materialize). Used by the
|
|
136
153
|
// runtime's auto-detail injection and the useEntityDetail kit hook.
|
|
137
154
|
// Pattern uses {slug} as the placeholder; substitution at runtime
|
|
138
155
|
// pulls from the dynamic-route param (entity-store) or the record's
|
|
139
|
-
// slug field (useEntityDetail).
|
|
156
|
+
// slug field (useEntityDetail). File-backed queries leave
|
|
140
157
|
// this null and get the static-file default /data/<name>/<slug>.json.
|
|
141
158
|
detailUrl: typeof config.detailUrl === 'string' ? config.detailUrl : null,
|
|
142
159
|
// `queryable:` declares the queryable surface — which fields a
|
|
143
160
|
// foundation can offer for filtering UI, with their type and
|
|
144
161
|
// type-specific metadata (enum options, range bounds). Foundations
|
|
145
|
-
// read this metadata via the kit's
|
|
162
|
+
// read this metadata via the kit's useQueryable hook to
|
|
146
163
|
// render filter controls and compose where-objects from user
|
|
147
164
|
// interactions. The framework doesn't validate the shape here —
|
|
148
165
|
// foundations get whatever the author wrote; documentation defines
|
|
@@ -170,7 +187,7 @@ function parseCollectionConfig(name, config) {
|
|
|
170
187
|
* fields and moved the page from /blog/docs-sites to /blog/11_docs_sites. The
|
|
171
188
|
* only trace was
|
|
172
189
|
*
|
|
173
|
-
* [
|
|
190
|
+
* [query-processor] YAML parse error: bad indentation of a mapping entry (4:72)
|
|
174
191
|
*
|
|
175
192
|
* on line 16 of 857 lines of build output, naming no file, nine lines above
|
|
176
193
|
* "Processed articles: 6 items" — a success line that reads as everything
|
|
@@ -303,20 +320,26 @@ function isExternalUrl(src) {
|
|
|
303
320
|
/**
|
|
304
321
|
* Process assets in collection content
|
|
305
322
|
* - Resolves relative paths to site-root-relative paths
|
|
306
|
-
* - Copies co-located assets to public/
|
|
323
|
+
* - Copies co-located assets to public/records/<schema>/
|
|
307
324
|
* - Updates paths in the content in place
|
|
308
325
|
*
|
|
309
326
|
* @param {Object} content - ProseMirror document
|
|
310
327
|
* @param {string} itemPath - Path to the markdown file
|
|
311
328
|
* @param {string} siteRoot - Site root directory
|
|
312
|
-
* @param {string}
|
|
329
|
+
* @param {string} queryName - Name of the collection (e.g., 'articles')
|
|
313
330
|
* @returns {Promise<Object>} Asset manifest for this item
|
|
314
331
|
*/
|
|
315
|
-
async function
|
|
332
|
+
async function processRecordAssets(content, itemPath, siteRoot, poolDirs, basePath) {
|
|
316
333
|
const assets = {}
|
|
317
334
|
const itemDir = dirname(itemPath)
|
|
318
335
|
const publicDir = join(siteRoot, 'public')
|
|
319
|
-
|
|
336
|
+
// ⭐ THE RECORD'S OWN HOME, keyed by its pool position — `public/records/<schema
|
|
337
|
+
// dirs>/`. It was `public/collections/<queryName>/`, which meant the SAME image
|
|
338
|
+
// was copied once per query that returned the record, under two URLs. Third
|
|
339
|
+
// instance of the same conflation (after the freeform locale tree and the
|
|
340
|
+
// translation manifest): an asset belongs to a record, and which query selects
|
|
341
|
+
// it is not a fact about it.
|
|
342
|
+
const targetDir = join(publicDir, 'records', poolDirs)
|
|
320
343
|
|
|
321
344
|
// Walk content and collect asset paths
|
|
322
345
|
const assetNodes = []
|
|
@@ -349,7 +372,7 @@ async function processCollectionAssets(content, itemPath, siteRoot, collectionNa
|
|
|
349
372
|
await copyFile(result.resolved, targetPath)
|
|
350
373
|
|
|
351
374
|
// Update path to site-root-relative
|
|
352
|
-
finalPath = `${basePath}
|
|
375
|
+
finalPath = `${basePath}records/${poolDirs}/${assetFilename}`
|
|
353
376
|
|
|
354
377
|
assets[src] = {
|
|
355
378
|
original: src,
|
|
@@ -384,7 +407,7 @@ async function processCollectionAssets(content, itemPath, siteRoot, collectionNa
|
|
|
384
407
|
const posterTarget = join(targetDir, posterFilename)
|
|
385
408
|
await mkdir(targetDir, { recursive: true })
|
|
386
409
|
await copyFile(posterResult.resolved, posterTarget)
|
|
387
|
-
node.attrs.poster = `${basePath}
|
|
410
|
+
node.attrs.poster = `${basePath}records/${poolDirs}/${posterFilename}`
|
|
388
411
|
}
|
|
389
412
|
}
|
|
390
413
|
|
|
@@ -395,7 +418,7 @@ async function processCollectionAssets(content, itemPath, siteRoot, collectionNa
|
|
|
395
418
|
const previewTarget = join(targetDir, previewFilename)
|
|
396
419
|
await mkdir(targetDir, { recursive: true })
|
|
397
420
|
await copyFile(previewResult.resolved, previewTarget)
|
|
398
|
-
node.attrs.preview = `${basePath}
|
|
421
|
+
node.attrs.preview = `${basePath}records/${poolDirs}/${previewFilename}`
|
|
399
422
|
}
|
|
400
423
|
}
|
|
401
424
|
}
|
|
@@ -406,17 +429,17 @@ async function processCollectionAssets(content, itemPath, siteRoot, collectionNa
|
|
|
406
429
|
/**
|
|
407
430
|
* Process assets in a data item (YAML/JSON)
|
|
408
431
|
* - Recursively walks the data object looking for local asset paths
|
|
409
|
-
* - Copies co-located assets to public/
|
|
432
|
+
* - Copies co-located assets to public/records/<schema>/
|
|
410
433
|
* - Rewrites paths to absolute URLs (with base path)
|
|
411
434
|
*
|
|
412
435
|
* @param {Object} data - Parsed data object (mutated in place)
|
|
413
436
|
* @param {string} itemPath - Path to the data file
|
|
414
437
|
* @param {string} siteRoot - Site root directory
|
|
415
|
-
* @param {string}
|
|
438
|
+
* @param {string} queryName - Name of the collection
|
|
416
439
|
* @param {string} basePath - Site base path (e.g., '/' or '/docs/')
|
|
417
440
|
*/
|
|
418
|
-
async function processDataItemAssets(data, itemPath, siteRoot,
|
|
419
|
-
const targetDir = join(siteRoot, 'public', '
|
|
441
|
+
async function processDataItemAssets(data, itemPath, siteRoot, poolDirs, basePath) {
|
|
442
|
+
const targetDir = join(siteRoot, 'public', 'records', poolDirs)
|
|
420
443
|
|
|
421
444
|
async function walk(parent, key) {
|
|
422
445
|
const val = parent[key]
|
|
@@ -427,7 +450,7 @@ async function processDataItemAssets(data, itemPath, siteRoot, collectionName, b
|
|
|
427
450
|
const filename = basename(resolved)
|
|
428
451
|
await mkdir(targetDir, { recursive: true })
|
|
429
452
|
await copyFile(resolved, join(targetDir, filename))
|
|
430
|
-
parent[key] = `${basePath}
|
|
453
|
+
parent[key] = `${basePath}records/${poolDirs}/${filename}`
|
|
431
454
|
}
|
|
432
455
|
} else if (val.startsWith('/')) {
|
|
433
456
|
// Absolute site path — just prepend base
|
|
@@ -467,7 +490,7 @@ async function processDataItemAssets(data, itemPath, siteRoot, collectionName, b
|
|
|
467
490
|
* @param {string} filename - YAML filename (.yml or .yaml)
|
|
468
491
|
* @returns {Promise<Object|Array|null>} Processed item(s) or null if unpublished
|
|
469
492
|
*/
|
|
470
|
-
async function processDataItem(dir, filename, siteRoot,
|
|
493
|
+
async function processDataItem(dir, filename, siteRoot, poolDirs, basePath) {
|
|
471
494
|
const filepath = join(dir, filename)
|
|
472
495
|
const raw = await readFile(filepath, 'utf-8')
|
|
473
496
|
const data = yaml.load(raw) || {}
|
|
@@ -476,7 +499,7 @@ async function processDataItem(dir, filename, siteRoot, collectionName, basePath
|
|
|
476
499
|
if (Array.isArray(data)) {
|
|
477
500
|
for (const item of data) {
|
|
478
501
|
if (item && typeof item === 'object') {
|
|
479
|
-
await processDataItemAssets(item, filepath, siteRoot,
|
|
502
|
+
await processDataItemAssets(item, filepath, siteRoot, poolDirs, basePath)
|
|
480
503
|
}
|
|
481
504
|
}
|
|
482
505
|
return data
|
|
@@ -486,7 +509,7 @@ async function processDataItem(dir, filename, siteRoot, collectionName, basePath
|
|
|
486
509
|
if (data.published === false) return null
|
|
487
510
|
const slug = basename(filename, extname(filename))
|
|
488
511
|
const item = { slug, ...data }
|
|
489
|
-
await processDataItemAssets(item, filepath, siteRoot,
|
|
512
|
+
await processDataItemAssets(item, filepath, siteRoot, poolDirs, basePath)
|
|
490
513
|
return item
|
|
491
514
|
}
|
|
492
515
|
|
|
@@ -501,7 +524,7 @@ async function processDataItem(dir, filename, siteRoot, collectionName, basePath
|
|
|
501
524
|
* @param {string} filename - JSON filename
|
|
502
525
|
* @returns {Promise<Object|Array|null>} Processed item(s) or null if unpublished
|
|
503
526
|
*/
|
|
504
|
-
async function processJsonItem(dir, filename, siteRoot,
|
|
527
|
+
async function processJsonItem(dir, filename, siteRoot, poolDirs, basePath) {
|
|
505
528
|
const filepath = join(dir, filename)
|
|
506
529
|
const raw = await readFile(filepath, 'utf-8')
|
|
507
530
|
const slug = basename(filename, '.json')
|
|
@@ -511,7 +534,7 @@ async function processJsonItem(dir, filename, siteRoot, collectionName, basePath
|
|
|
511
534
|
if (Array.isArray(data)) {
|
|
512
535
|
for (const item of data) {
|
|
513
536
|
if (item && typeof item === 'object') {
|
|
514
|
-
await processDataItemAssets(item, filepath, siteRoot,
|
|
537
|
+
await processDataItemAssets(item, filepath, siteRoot, poolDirs, basePath)
|
|
515
538
|
}
|
|
516
539
|
}
|
|
517
540
|
return data
|
|
@@ -520,7 +543,7 @@ async function processJsonItem(dir, filename, siteRoot, collectionName, basePath
|
|
|
520
543
|
// Object → single item
|
|
521
544
|
if (data.published === false) return null
|
|
522
545
|
const item = { slug, ...data }
|
|
523
|
-
await processDataItemAssets(item, filepath, siteRoot,
|
|
546
|
+
await processDataItemAssets(item, filepath, siteRoot, poolDirs, basePath)
|
|
524
547
|
return item
|
|
525
548
|
}
|
|
526
549
|
|
|
@@ -556,7 +579,7 @@ async function processBibtexItem(dir, filename) {
|
|
|
556
579
|
* @param {string} siteRoot - Site root directory for asset resolution
|
|
557
580
|
* @returns {Promise<Object|null>} Processed item or null if unpublished
|
|
558
581
|
*/
|
|
559
|
-
async function processContentItem(dir, filename, config, siteRoot, basePath) {
|
|
582
|
+
async function processContentItem(dir, filename, config, siteRoot, basePath, poolDirs) {
|
|
560
583
|
const filepath = join(dir, filename)
|
|
561
584
|
const raw = await readFile(filepath, 'utf-8')
|
|
562
585
|
const slug = basename(filename, extname(filename))
|
|
@@ -574,13 +597,13 @@ async function processContentItem(dir, filename, config, siteRoot, basePath) {
|
|
|
574
597
|
|
|
575
598
|
// Process assets (resolve paths, copy co-located files)
|
|
576
599
|
// This modifies content in place, updating paths to site-root-relative
|
|
577
|
-
await
|
|
600
|
+
await processRecordAssets(content, filepath, siteRoot, poolDirs, basePath)
|
|
578
601
|
|
|
579
602
|
// Extract excerpt
|
|
580
603
|
const excerpt = extractExcerpt(frontmatter, content, config.excerpt)
|
|
581
604
|
|
|
582
605
|
// Extract first image (frontmatter takes precedence)
|
|
583
|
-
// Note: paths in content have already been updated by
|
|
606
|
+
// Note: paths in content have already been updated by processRecordAssets
|
|
584
607
|
const image = frontmatter.image || extractFirstImage(content)
|
|
585
608
|
|
|
586
609
|
return {
|
|
@@ -631,7 +654,7 @@ async function collectSourceFiles(dir, rel = '') {
|
|
|
631
654
|
* whichever sorts last wins the route and the per-record file. That is a real
|
|
632
655
|
* ambiguity only the author can resolve, so it is reported rather than repaired.
|
|
633
656
|
*/
|
|
634
|
-
function warnDuplicateSlugs(items,
|
|
657
|
+
function warnDuplicateSlugs(items, queryName) {
|
|
635
658
|
const seen = new Map()
|
|
636
659
|
for (const item of items) {
|
|
637
660
|
if (!item || item.slug === undefined) continue
|
|
@@ -639,7 +662,7 @@ function warnDuplicateSlugs(items, collectionName) {
|
|
|
639
662
|
const where = item.path ? `${item.path}/` : ''
|
|
640
663
|
if (seen.has(slug)) {
|
|
641
664
|
console.warn(
|
|
642
|
-
`[
|
|
665
|
+
`[query-processor] Query "${queryName}" has more than one record with ` +
|
|
643
666
|
`slug "${slug}" (${seen.get(slug)}${slug}, ${where}${slug}). Its detail route and ` +
|
|
644
667
|
`per-record file resolve to only one of them — give them distinct slugs.`
|
|
645
668
|
)
|
|
@@ -656,41 +679,49 @@ function warnDuplicateSlugs(items, collectionName) {
|
|
|
656
679
|
* @param {Object} config - Parsed collection config
|
|
657
680
|
* @returns {Promise<Array>} Array of processed items
|
|
658
681
|
*/
|
|
659
|
-
async function collectItems(siteDir, config,
|
|
660
|
-
|
|
661
|
-
|
|
662
|
-
|
|
663
|
-
//
|
|
664
|
-
|
|
665
|
-
|
|
666
|
-
|
|
667
|
-
|
|
668
|
-
|
|
669
|
-
|
|
670
|
-
|
|
671
|
-
// Process all collection files (markdown → content items, YAML/JSON → data
|
|
672
|
-
// items, BibTeX → CSL-JSON bibliography items).
|
|
682
|
+
async function collectItems(siteDir, config, entitiesDir, basePath) {
|
|
683
|
+
// ⭐ THE QUERY NAMES A SCHEMA AND THE POOL FOLLOWS — the same resolution the
|
|
684
|
+
// sync lane makes, from the same reader, so the two lanes cannot disagree
|
|
685
|
+
// about which files are a query's records. They used to: this one recursed
|
|
686
|
+
// into a collection directory and sync did not.
|
|
687
|
+
const pooled = config.poolEntities || []
|
|
688
|
+
if (pooled.length === 0) return []
|
|
689
|
+
|
|
690
|
+
const dirOf = (e) => resolve(siteDir, entitiesDir || ENTITIES_DIR, ...e.dirs)
|
|
691
|
+
|
|
692
|
+
// Process all entity files (markdown → content items, YAML/JSON → data items,
|
|
693
|
+
// BibTeX → CSL-JSON bibliography items).
|
|
673
694
|
let items = await Promise.all(
|
|
674
|
-
|
|
675
|
-
|
|
676
|
-
|
|
695
|
+
pooled.map((e) => {
|
|
696
|
+
const dir = dirOf(e)
|
|
697
|
+
const file = `${e.slug}${e.ext}`
|
|
698
|
+
if (e.ext === '.bib') {
|
|
699
|
+
return processBibtexItem(dir, file)
|
|
677
700
|
}
|
|
678
|
-
if (
|
|
679
|
-
return processJsonItem(
|
|
701
|
+
if (e.ext === '.json') {
|
|
702
|
+
return processJsonItem(dir, file, siteDir, e.dirs.join('/'), basePath)
|
|
680
703
|
}
|
|
681
|
-
if (
|
|
682
|
-
return processDataItem(
|
|
704
|
+
if (e.ext === '.yml' || e.ext === '.yaml') {
|
|
705
|
+
return processDataItem(dir, file, siteDir, e.dirs.join('/'), basePath)
|
|
683
706
|
}
|
|
684
|
-
return processContentItem(
|
|
707
|
+
return processContentItem(dir, file, config, siteDir, basePath, e.dirs.join('/'))
|
|
685
708
|
})
|
|
686
709
|
)
|
|
687
710
|
|
|
688
|
-
//
|
|
689
|
-
//
|
|
690
|
-
//
|
|
711
|
+
// ⭐ `path` IS THE PLACEMENT `records.yml` GAVE THE RECORD, and it is the whole
|
|
712
|
+
// reason folders exist: `where: { path: { under: 'archive' } }` is how a query
|
|
713
|
+
// asks for a slice. Structure is query scope, not navigation.
|
|
714
|
+
//
|
|
715
|
+
// ⛔ THIS WAS HARDCODED TO `''` FOR A WHILE, AND THE COMMENT SAID "until the
|
|
716
|
+
// folder producer lands". It landed, and this was not revisited — so every
|
|
717
|
+
// folder slice matched NOTHING on the delivery lane, silently, which is the one
|
|
718
|
+
// failure mode the whole design is built to prevent. Measured before the fix: a
|
|
719
|
+
// two-record site with an `archive` folder returned `[]` for its own slice.
|
|
720
|
+
//
|
|
721
|
+
// ⚠️ It stays a SCALAR. `matchUnder` in `core/src/where.js` is string-only, so
|
|
722
|
+
// an array would match nothing — one placement per entity is the ruling.
|
|
691
723
|
items = items.map((result, i) => {
|
|
692
|
-
const
|
|
693
|
-
const path = dir === '.' ? '' : dir
|
|
724
|
+
const path = pooled[i] ? (config.placements?.get(pooled[i].id)?.path ?? '') : ''
|
|
694
725
|
if (Array.isArray(result)) return result.map((item) => item && { ...item, path })
|
|
695
726
|
return result && { ...result, path }
|
|
696
727
|
})
|
|
@@ -713,7 +744,21 @@ async function collectItems(siteDir, config, collectionsBase, basePath) {
|
|
|
713
744
|
}))
|
|
714
745
|
}
|
|
715
746
|
|
|
716
|
-
//
|
|
747
|
+
// ⛔ ORDER MATCHES `data-fetcher.js::applyPostProcessing` — where, filter, sort,
|
|
748
|
+
// limit. Two lanes evaluate the same declaration (this one materializes a query
|
|
749
|
+
// to `/data/<name>.json`; that one runs a page-level `fetch:`), so a difference
|
|
750
|
+
// in order is a difference in RESULT for any query that both narrows and limits.
|
|
751
|
+
//
|
|
752
|
+
// ⚠️ `where` was missing here entirely until 2026-08-29: `parseQueryConfig`
|
|
753
|
+
// read `filter` and never `where`, so the CANONICAL predicate was parsed, put on
|
|
754
|
+
// the sync wire, stored — and never applied, while the DEPRECATED one it replaced
|
|
755
|
+
// worked. An author following current guidance got silence and shipped unfiltered
|
|
756
|
+
// data. Pinned by `tests/collection-query-terms.test.js`.
|
|
757
|
+
if (config.where) {
|
|
758
|
+
items = applyWhere(items, config.where)
|
|
759
|
+
}
|
|
760
|
+
|
|
761
|
+
// Apply the legacy filter expression (deprecated)
|
|
717
762
|
if (config.filter) {
|
|
718
763
|
items = applyFilter(items, config.filter)
|
|
719
764
|
}
|
|
@@ -735,28 +780,68 @@ async function collectItems(siteDir, config, collectionsBase, basePath) {
|
|
|
735
780
|
* Process all content collections defined in site.yml
|
|
736
781
|
*
|
|
737
782
|
* @param {string} siteDir - Site root directory
|
|
738
|
-
* @param {Object}
|
|
783
|
+
* @param {Object} queriesConfig - the resolved QUERY declarations
|
|
784
|
+
* @param {string} [entitiesDir] - pool directory override (`site.yml::paths.entities`)
|
|
739
785
|
* @returns {Promise<Object>} Map of collection name to items array
|
|
740
786
|
*
|
|
741
787
|
* @example
|
|
742
|
-
* const collections = await
|
|
788
|
+
* const collections = await processQueries('/path/to/site', {
|
|
743
789
|
* articles: { path: 'collections/articles', sort: 'date desc' },
|
|
744
790
|
* products: 'collections/products'
|
|
745
791
|
* })
|
|
746
792
|
* // { articles: [...], products: [...] }
|
|
747
793
|
*/
|
|
748
|
-
export async function
|
|
749
|
-
if (!
|
|
794
|
+
export async function processQueries(siteDir, queriesConfig, entitiesDir, basePath = '/') {
|
|
795
|
+
if (!queriesConfig || typeof queriesConfig !== 'object') {
|
|
750
796
|
return {}
|
|
751
797
|
}
|
|
752
798
|
|
|
799
|
+
// ⭐ ONE POOL WALK FOR EVERY QUERY. Two queries over the same schema read one
|
|
800
|
+
// set of files; a query reads none of another schema's.
|
|
801
|
+
const pool = await readEntityPool(siteDir, { dir: entitiesDir })
|
|
802
|
+
if (pool.errors.length) {
|
|
803
|
+
for (const e of pool.errors) console.warn(`[query-processor] ${e}`)
|
|
804
|
+
}
|
|
805
|
+
|
|
806
|
+
// ⛔ `records.yml` DECIDES WHAT IS PUBLISHED ON THIS LANE TOO, and it did not
|
|
807
|
+
// until now. Only the sync lane honoured it, so removing a record from
|
|
808
|
+
// `records.yml` left it shipping in `/data/<name>.json` on every static host —
|
|
809
|
+
// an author unpublishes a draft and it stays public. Measured before the fix.
|
|
810
|
+
//
|
|
811
|
+
// ⚖️ MISSING IS NOT EMPTY HERE EITHER, but it means something different from
|
|
812
|
+
// what it means to sync. There is no server folder to leave alone, so a site
|
|
813
|
+
// with no `records.yml` is simply not managing publication, and its whole pool
|
|
814
|
+
// is delivered. (Making missing mean "publish nothing" would turn every site
|
|
815
|
+
// without the file into a silently empty one.)
|
|
816
|
+
const recordsCfg = await readRecordsConfig(siteDir)
|
|
817
|
+
if (recordsCfg.error) console.warn(`[query-processor] ${recordsCfg.error}`)
|
|
818
|
+
const managed = recordsCfg.state !== FOLDER_MISSING
|
|
819
|
+
const folder = managed ? resolveFolder(recordsCfg.entries, pool.entities) : null
|
|
820
|
+
if (folder) {
|
|
821
|
+
for (const e of folder.errors) console.error(`[query-processor] ${e}`)
|
|
822
|
+
}
|
|
823
|
+
const published = folder
|
|
824
|
+
? pool.entities.filter((e) => folder.placements.has(e.id))
|
|
825
|
+
: pool.entities
|
|
826
|
+
|
|
827
|
+
const poolBySchema = groupPoolBySchema(published)
|
|
828
|
+
|
|
753
829
|
const results = {}
|
|
754
830
|
|
|
755
|
-
for (const [name, config] of Object.entries(
|
|
756
|
-
const parsed =
|
|
757
|
-
|
|
831
|
+
for (const [name, config] of Object.entries(queriesConfig)) {
|
|
832
|
+
const parsed = parseQueryConfig(name, config)
|
|
833
|
+
parsed.poolEntities = parsed.schema ? poolBySchema.get(parsed.schema) || [] : []
|
|
834
|
+
parsed.placements = folder?.placements ?? null
|
|
835
|
+
if (parsed.poolEntities.length === 0 && !parsed.url) {
|
|
836
|
+
console.warn(
|
|
837
|
+
`[query-processor] Query "${name}" matches no records — nothing ` +
|
|
838
|
+
`published declares ${parsed.schema || '(no schema)'}. ` +
|
|
839
|
+
(managed ? 'Check records.yml lists them.' : 'Check entities/.')
|
|
840
|
+
)
|
|
841
|
+
}
|
|
842
|
+
const items = await collectItems(siteDir, parsed, entitiesDir, basePath)
|
|
758
843
|
results[name] = items
|
|
759
|
-
console.log(`[
|
|
844
|
+
console.log(`[query-processor] Processed ${name}: ${items.length} items`)
|
|
760
845
|
}
|
|
761
846
|
|
|
762
847
|
return results
|
|
@@ -802,7 +887,7 @@ async function pruneOrphanedRecords(dataDir, name, expected) {
|
|
|
802
887
|
const contained = resolve(recordsDir)
|
|
803
888
|
if (contained !== resolve(dataDir, name) || !contained.startsWith(resolve(dataDir) + sep)) {
|
|
804
889
|
console.warn(
|
|
805
|
-
`[
|
|
890
|
+
`[query-processor] Refusing to prune "${name}" — it does not resolve ` +
|
|
806
891
|
`inside ${dataDir}`
|
|
807
892
|
)
|
|
808
893
|
return []
|
|
@@ -826,22 +911,22 @@ async function pruneOrphanedRecords(dataDir, name, expected) {
|
|
|
826
911
|
* @returns {Promise<void>}
|
|
827
912
|
*
|
|
828
913
|
* @example
|
|
829
|
-
* await
|
|
914
|
+
* await writeQueryFiles('/path/to/site', {
|
|
830
915
|
* articles: [{ slug: 'hello', title: 'Hello World', ... }]
|
|
831
916
|
* })
|
|
832
917
|
* // Creates public/data/articles.json
|
|
833
918
|
*/
|
|
834
|
-
export async function
|
|
835
|
-
if (!
|
|
919
|
+
export async function writeQueryFiles(siteDir, byQuery, queriesConfig = null) {
|
|
920
|
+
if (!byQuery || Object.keys(byQuery).length === 0) {
|
|
836
921
|
return
|
|
837
922
|
}
|
|
838
923
|
|
|
839
924
|
const dataDir = join(siteDir, 'public', DATA_DIR)
|
|
840
925
|
await mkdir(dataDir, { recursive: true })
|
|
841
926
|
|
|
842
|
-
for (const [name, items] of Object.entries(
|
|
843
|
-
const rawConfig =
|
|
844
|
-
const parsed = rawConfig ?
|
|
927
|
+
for (const [name, items] of Object.entries(byQuery)) {
|
|
928
|
+
const rawConfig = queriesConfig?.[name]
|
|
929
|
+
const parsed = rawConfig ? parseQueryConfig(name, rawConfig) : null
|
|
845
930
|
const deferred = parsed?.deferred
|
|
846
931
|
|
|
847
932
|
if (deferred && deferred.length > 0) {
|
|
@@ -872,21 +957,21 @@ export async function writeCollectionFiles(siteDir, collections, collectionsConf
|
|
|
872
957
|
const cascadePath = join(dataDir, `${name}.json`)
|
|
873
958
|
await writeFile(cascadePath, JSON.stringify(stripped, null, 2))
|
|
874
959
|
console.log(
|
|
875
|
-
`[
|
|
960
|
+
`[query-processor] Generated ${cascadePath} (${items.length} items, ` +
|
|
876
961
|
`deferred: [${deferred.join(', ')}]) + ${perRecordCount} per-record files`
|
|
877
962
|
)
|
|
878
963
|
if (pruned.length > 0) {
|
|
879
964
|
// A deletion is always worth naming. These files were public a moment
|
|
880
965
|
// ago, so "which ones went" is the question an author will have.
|
|
881
966
|
console.log(
|
|
882
|
-
`[
|
|
967
|
+
`[query-processor] Removed ${pruned.length} stale per-record ` +
|
|
883
968
|
`file(s) from ${recordsDir}: ${pruned.join(', ')}`
|
|
884
969
|
)
|
|
885
970
|
}
|
|
886
971
|
} else {
|
|
887
972
|
const filepath = join(dataDir, `${name}.json`)
|
|
888
973
|
await writeFile(filepath, JSON.stringify(items, null, 2))
|
|
889
|
-
console.log(`[
|
|
974
|
+
console.log(`[query-processor] Generated ${filepath} (${items.length} items)`)
|
|
890
975
|
|
|
891
976
|
// This collection is not deferred, so it has no per-record files. If it
|
|
892
977
|
// used to, the directory is still there and will never be written again
|
|
@@ -894,7 +979,7 @@ export async function writeCollectionFiles(siteDir, collections, collectionsConf
|
|
|
894
979
|
const pruned = await pruneOrphanedRecords(dataDir, name, new Set())
|
|
895
980
|
if (pruned.length > 0) {
|
|
896
981
|
console.log(
|
|
897
|
-
`[
|
|
982
|
+
`[query-processor] Removed ${pruned.length} per-record file(s) ` +
|
|
898
983
|
`from ${join(dataDir, name)} — "${name}" no longer declares deferred:`
|
|
899
984
|
)
|
|
900
985
|
}
|
|
@@ -909,15 +994,15 @@ export async function writeCollectionFiles(siteDir, collections, collectionsConf
|
|
|
909
994
|
* @param {Object} config - Collection config
|
|
910
995
|
* @returns {Promise<Date|null>} Most recent modification time
|
|
911
996
|
*/
|
|
912
|
-
export async function
|
|
913
|
-
const parsed =
|
|
914
|
-
const
|
|
997
|
+
export async function getQueryLastModified(siteDir, config) {
|
|
998
|
+
const parsed = parseQueryConfig('temp', config)
|
|
999
|
+
const poolDir = join(siteDir, parsed.path)
|
|
915
1000
|
|
|
916
|
-
if (!existsSync(
|
|
1001
|
+
if (!existsSync(poolDir)) {
|
|
917
1002
|
return null
|
|
918
1003
|
}
|
|
919
1004
|
|
|
920
|
-
const files = await readdir(
|
|
1005
|
+
const files = await readdir(poolDir)
|
|
921
1006
|
const itemFiles = files.filter(f =>
|
|
922
1007
|
!f.startsWith('_') &&
|
|
923
1008
|
(f.endsWith('.md') || f.endsWith('.yml') || f.endsWith('.yaml') || f.endsWith('.json') || f.endsWith('.bib'))
|
|
@@ -926,7 +1011,7 @@ export async function getCollectionLastModified(siteDir, config) {
|
|
|
926
1011
|
let lastModified = null
|
|
927
1012
|
|
|
928
1013
|
for (const file of itemFiles) {
|
|
929
|
-
const fileStat = await stat(join(
|
|
1014
|
+
const fileStat = await stat(join(poolDir, file))
|
|
930
1015
|
if (!lastModified || fileStat.mtime > lastModified) {
|
|
931
1016
|
lastModified = fileStat.mtime
|
|
932
1017
|
}
|