@slybridges/kiss 0.8.5 → 0.9.0

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@slybridges/kiss",
-  "version": "0.8.5",
+  "version": "0.9.0",
   "description": "Keep It Simple and Static site generator",
   "main": "src/index.js",
   "bin": {
@@ -12,9 +12,9 @@
   "author": "Sylvestre Dupont",
   "license": "MIT",
   "dependencies": {
-    "browser-sync": "^3.0.2",
+    "browser-sync": "^3.0.3",
     "chalk": "^4.1.2",
-    "cheerio": "^1.0.0-rc.12",
+    "cheerio": "^1.0.0",
     "chokidar": "^3.6.0",
     "date-fns": "^3.6.0",
     "fast-glob": "^3.3.2",
@@ -23,17 +23,17 @@
     "lodash": "^4.17.21",
     "marked": "^12.0.1",
     "nunjucks": "^3.2.4",
-    "sharp": "^0.33.3",
+    "sharp": "^0.33.5",
     "slugify": "^1.6.6",
     "xml": "^1.0.1",
     "yargs": "^17.7.2"
   },
   "devDependencies": {
-    "@eslint/js": "^9.5.0",
-    "eslint": "^9.5.0",
+    "@eslint/js": "^9.11.1",
+    "eslint": "^9.11.1",
     "eslint-config-prettier": "^9.1.0",
-    "globals": "^15.6.0",
-    "prettier": "^3.2.5"
+    "globals": "^15.9.0",
+    "prettier": "^3.3.3"
   },
   "repository": {
     "type": "git",

package/src/build.js

@@ -5,62 +5,115 @@ const fg = require("fast-glob")
 const path = require("path")
 
 const { loadConfig } = require("./config")
-const { getParentId, relativeToAbsoluteAttributes } = require("./helpers")
+const {
+  computeParentId,
+  getBuildEntries,
+  getPageFromInputPath,
+  relativeToAbsoluteAttributes,
+} = require("./helpers")
 const { baseLoader } = require("./loaders")
 const { setGlobalLogger } = require("./logger")
 
-const build = async (options = {}, config = null) => {
+const hasOwnProperty = Object.prototype.hasOwnProperty
+
+const build = async (options = {}, lastBuild = {}, version = 0) => {
   console.time("Build time")
 
   const { configFile, unsafeBuild, verbosity, watchMode } = options
 
   setGlobalLogger(verbosity)
 
-  let context = {
-    pages: {},
-  }
+  let { config } = lastBuild
 
-  global.logger.section("Loading config and initial context")
   if (!config) {
+    global.logger.section("Loading config and initial context")
+    config = loadConfig({ configFile })
+  }
+
+  // config file can either be passed as option or be the default in the last build config
+  const actualConfigFile = configFile || config?.configFile
+  if (options.incremental && options.file === actualConfigFile) {
+    global.logger.section("Reloading config and initial context")
     config = loadConfig({ configFile })
+    // clearing last build contect in case of config change
+    lastBuild.context = null
   }
-  context = { ...context, ...config.context }
 
-  if (config.hooks.loadLibs.length > 0) {
+  const buildFlags = computeBuildFlags(
+    options,
+    config,
+    lastBuild.context,
+    version,
+  )
+
+  let context = lastBuild.context || { pages: {}, ...config.context }
+
+  if (options.incremental) {
+    // compute the file ids that will need to be updated
+    buildFlags.buildPageIds = computeBuildPageIDs(context, buildFlags)
+    if (buildFlags.buildPageIds.length > 0) {
+      global.logger.info(
+        "Change impacts pages with IDs",
+        buildFlags.buildPageIds,
+      )
+    } else if (!lastBuild.context) {
+      global.logger.info(
+        "Incremental build mode. Performing initial full build...",
+      )
+    } else {
+      global.logger.info(
+        "Incremental rebuild not possible. Performing full rebuild...",
+      )
+    }
+  }
+
+  if (buildFlags.loadLibs) {
     global.logger.section("Running loadLibs hooks")
-    config = runConfigHooks(config, "loadLibs")
+    config = runConfigHooks(config, "loadLibs", null, buildFlags)
   }
 
-  if (config.hooks.preLoad.length > 0) {
+  if (buildFlags.preLoad) {
     global.logger.section("Running preLoad hooks")
-    context = runConfigHooks(config, "preLoad", context)
+    context = runConfigHooks(config, "preLoad", context, buildFlags)
   }
 
-  global.logger.section(
-    `Loading content from '${config.dirs.content}' directory`,
-  )
-  context.pages = await loadContent(config, context)
+  if (buildFlags.content) {
+    if (options.incremental && buildFlags.buildPageIds.length > 0) {
+      global.logger.section("Reloading content")
+    } else {
+      global.logger.section(`Loading content from '${config.dirs.content}'`)
+    }
+    context.pages = await loadContent(config, context, buildFlags)
+  }
 
-  if (config.hooks.postLoad.length > 0) {
+  if (buildFlags.postLoad) {
     global.logger.section("Running postLoad hooks")
-    context = runConfigHooks(config, "postLoad", context)
+    context = runConfigHooks(config, "postLoad", context, buildFlags)
   }
 
-  global.logger.section("Computing dynamic page context")
-  context = computeAllPagesData(context, config)
+  if (buildFlags.dynamicData) {
+    global.logger.section("Computing dynamic page context")
+    context = computeAllPagesData(context, config, buildFlags)
+  }
 
-  global.logger.section("Computing data views")
-  context = computeDataViews(context, config)
+  if (buildFlags.dataViews) {
+    global.logger.section("Computing data views")
+    context = computeDataViews(context, config)
+  }
 
-  global.logger.section(`Applying transforms`)
-  context = await applyTransforms(context, config)
+  if (buildFlags.transform) {
+    global.logger.section(`Applying transforms`)
+    context = await applyTransforms(context, config, buildFlags)
+  }
 
-  global.logger.section(`Writing site to '${config.dirs.public}' directory`)
-  await writeStaticSite(context, config)
+  if (buildFlags.write) {
+    global.logger.section(`Writing site to '${config.dirs.public}' directory`)
+    await writeStaticSite(context, config, buildFlags)
+  }
 
-  if (config.hooks.postWrite.length > 0) {
+  if (buildFlags.postWrite) {
     global.logger.section("Running postWrite hooks")
-    runConfigHooks(config, "postWrite", context)
+    runConfigHooks(config, "postWrite", context, buildFlags)
   }
 
   global.logger.section("Build complete")
@@ -84,16 +137,19 @@ const build = async (options = {}, config = null) => {
   } else if (warningCount > 0) {
     global.logger.warn(`${warningCount} warning(s) found.`)
   } else {
-    global.logger.success(`Static build completed!`)
+    global.logger.success(`Build completed!`)
   }
   console.timeEnd("Build time")
+
+  return { context, config }
 }
 
 module.exports = build
 
 /** Private **/
 
-const applyTransforms = async (context, config) => {
+const applyTransforms = async (context, config, buildFlags) => {
+  const { buildPageIds, incremental } = buildFlags
   if (!config.transforms || config.transforms.length === 0) {
     global.logger.info(`No transform registered.`)
     return context
@@ -119,7 +175,7 @@ const applyTransforms = async (context, config) => {
         options.description || `Transforming context using ${handler.name}`
       global.logger.info(message)
       try {
-        context = await handler(context, options, config)
+        context = await handler(context, options, config, buildFlags)
       } catch (err) {
         global.logger.error(
           `[${handler.name}] Error during transform:\n`,
@@ -127,12 +183,21 @@ const applyTransforms = async (context, config) => {
         )
       }
     } else {
-      // page transforms
-      const message =
+      const entries = getBuildEntries(context, buildFlags).filter(
+        ([, page]) => !outputType || outputType === page._meta.outputType,
+      )
+      if (entries.length === 0) {
+        continue
+      }
+      let message =
         options.description ||
-        `Transforming ${outputType || "all"} pages using ${handler.name}`
+        `Transforming ${outputType || "all"} pages using '${handler.name}'`
+      if (incremental && buildPageIds.length > 0) {
+        message += ` (incremental)`
+      }
       global.logger.info(message)
-      for (let [id, page] of Object.entries(context.pages)) {
+      // page transforms
+      for (let [id, page] of entries) {
         if (outputType && page._meta.outputType !== outputType) {
           continue
         }
@@ -151,6 +216,140 @@ const applyTransforms = async (context, config) => {
   return context
 }
 
+const computeBuildFlags = (options, config, lastContext, version) => {
+  let flags = {
+    incremental: options.incremental,
+    config: true,
+    loadLibs: config.hooks.loadLibs.length > 0,
+    preLoad: config.hooks.preLoad.length > 0,
+    content: true,
+    postLoad: config.hooks.postLoad.length > 0,
+    dynamicData: true,
+    dataViews: true,
+    transform: true,
+    write: true,
+    postWrite: config.hooks.postWrite.length > 0,
+    version: version,
+  }
+  if (!options.incremental || !lastContext || !options.file) {
+    // full build
+    return flags
+  }
+  // incremental build mode
+  if (options.event === "addDir") {
+    // skip directory changes
+    return {
+      incremental: options.incremental,
+      config: false,
+      loadLibs: false,
+      preLoad: false,
+      content: false,
+      postLoad: false,
+      dynamicData: false,
+      dataViews: false,
+      transform: false,
+      write: false,
+      postWrite: false,
+      version: version,
+    }
+  }
+  // only 'change' event is supported for now in incremental mode
+  // 'add', 'unlink' and 'unlinkDir' will trigger a full rebuild
+  if (options.event !== "change") {
+    return flags
+  }
+  flags.file = options.file
+  flags.config = false
+  flags.content = false
+  flags.dynamicData = false
+  flags.dataViews = false
+  if (options.file.startsWith(config.dirs.content)) {
+    // Content change: we need to reload the changed content, recompute the data,
+    // do the transforms and write the files
+    flags.content = true
+    flags.contentFile = options.file
+    flags.dynamicData = true
+    flags.dataViews = true
+  } else if (options.file.startsWith(config.dirs.template)) {
+    // Template change: no need to reload the content or recompute the data.
+    // We only need to perform the transforms and write the files
+    // Templates are always related to the template directory
+    flags.templateFile = path.relative(config.dirs.template, options.file)
+  }
+  // compute hook flags
+  flags.loadLibs = computeIncrementalHookBuildFlag(
+    config.hooks.loadLibs,
+    options.file,
+    lastContext,
+  )
+  flags.preLoad = computeIncrementalHookBuildFlag(
+    config.hooks.preLoad,
+    options.file,
+    lastContext,
+  )
+  flags.postLoad = computeIncrementalHookBuildFlag(
+    config.hooks.postLoad,
+    options.file,
+    lastContext,
+  )
+  flags.postWrite = computeIncrementalHookBuildFlag(
+    config.hooks.postWrite,
+    options.file,
+    lastContext,
+  )
+  // other change: full rebuild
+  return flags
+}
+
+// return the list of ids impacted by a file change which are
+// the file itself, its ascendants and descendants in case of a content file change
+// or all pages using a template in case of a template file change
+// if there is no file change, return an empty list
+const computeBuildPageIDs = (context, buildFlags) => {
+  let buildPageIds = []
+  if (buildFlags.contentFile) {
+    const page = getPageFromInputPath(buildFlags.contentFile, context.pages)
+    if (!page) {
+      global.logger.warn(
+        `Could not find existing page for '${buildFlags.contentFile}'.`,
+      )
+      return []
+    }
+    // ascendants
+    buildPageIds = buildPageIds.concat(page._meta.ascendants)
+    // current page
+    buildPageIds.push(page._meta.id)
+    // descendants
+    buildPageIds = buildPageIds.concat(page._meta.descendants)
+  } else if (buildFlags.templateFile) {
+    // mark all pages using that template for rebuild
+    // Note: if the template is a sub-template of another template
+    // (e.g. that is included() in a main template),
+    // this will not work and we'll rebuild everything
+    buildPageIds = Object.values(context.pages)
+      .filter((page) => page.layout === buildFlags.templateFile)
+      .map((page) => page._meta.id)
+  }
+  return buildPageIds
+}
+
+const computeIncrementalHookBuildFlag = (hooks, file, context) => {
+  if (!hooks || hooks.length === 0) {
+    return false
+  }
+  // check if some of the hook have a incrementalRebuild attribute
+  // if they do call the function and return true if any of the function returns true
+  return hooks.some((hook) => {
+    if (typeof hook === "function") {
+      return true
+    }
+    if (typeof hook.incrementalRebuild === "function") {
+      return hook.incrementalRebuild(file, context)
+    }
+    return false
+  })
+}
+
 // load content derived from existing pages
 const computeDataViews = (context, config) => {
   config.dataViews.forEach((view) => {
@@ -170,32 +369,28 @@ const computeDataViews = (context, config) => {
   return context
 }
 
-const computePageData = (data, config, context, options = {}) => {
+const computePageData = (data, config, context, buildFlags, options = {}) => {
   let computed = {
-    data: null,
+    data: _.isArray(data) ? [...data] : { ...data },
     pendingCount: 0,
   }
-  if (_.isArray(data)) {
-    computed.data = [...data]
-  } else {
-    computed.data = { ...data }
-  }
   if (!options.topLevelData) {
     // for recursive call
     options.topLevelData = data
   }
   for (let key in data) {
     let value = data[key]
-    //_.forEach(data, (value, key) => {
-    if (typeof key === "string" && key.endsWith("_no_cascade")) {
-      // this is an override key, computing original key value
-      key = key.split("_no_cascade")[0]
-    } else if (
-      Object.prototype.hasOwnProperty.call(data, key + "_no_cascade")
-    ) {
-      // there is a data override attribute, bail out.
-      continue
+
+    if (typeof key === "string") {
+      if (key.endsWith("_no_cascade")) {
+        // this is an override key, computing original key value
+        key = key.split("_no_cascade")[0]
+      } else if (hasOwnProperty.call(data, key + "_no_cascade")) {
+        // there is a data override attribute, bail out.
+        continue
+      }
     }
+
     if (typeof value === "function") {
       // it's a function we need to compute the result
       let currentPending = 0
@@ -219,7 +414,13 @@ const computePageData = (data, config, context, options = {}) => {
       }
     } else if (_.isPlainObject(value) || _.isArray(value)) {
       // it's an object: we need to see if there is data to compute inside
-      let subComputed = computePageData(value, config, context, options)
+      let subComputed = computePageData(
+        value,
+        config,
+        context,
+        buildFlags,
+        options,
+      )
       computed.data[key] = subComputed.data
       computed.pendingCount += subComputed.pendingCount
     } else {
@@ -229,16 +430,17 @@ const computePageData = (data, config, context, options = {}) => {
   return computed
 }
 
-const computeAllPagesData = (context, config) => {
+const computeAllPagesData = (context, config, buildFlags) => {
   let pendingTotal = 0
   let round = 1
   let computed = {}
 
   while (round === 1 || pendingTotal > 0) {
+    let entries = getBuildEntries(context, buildFlags)
     pendingTotal = 0
-    _.forEach(context.pages, (page, key) => {
+    entries.forEach(([key, page]) => {
       try {
-        computed = computePageData(page, config, context)
+        computed = computePageData(page, config, context, buildFlags)
       } catch (err) {
         global.logger.error(
           `[computePageData] Error during computing page data for page id '${page._meta.id}'\n`,
@@ -249,10 +451,9 @@ const computeAllPagesData = (context, config) => {
       pendingTotal += computed.pendingCount
     })
     if (pendingTotal > 0 && round + 1 > config.defaults.maxComputingRounds) {
-      global.logger.error(
-        ```Could not compute all data in ${config.defaults.maxComputingRounds} rounds. Check for circular
-dependencies or increase the 'maxComputingRounds' value.```,
-      )
+      let message = `Could not compute all data in ${config.defaults.maxComputingRounds} rounds.`
+      message += ` Check for circular dependencies or increase the 'config.defaults.maxComputingRounds' value.`
+      global.logger.error(message)
       break
     }
     if (pendingTotal > 0) {
@@ -310,32 +511,55 @@ const countPendingDependencies = (page, pages, deps = []) => {
   return pendingCount
 }
 
-const directoryCollectionLoader = (pathname, options, pages, config) => {
-  const parentName = path.dirname(pathname)
-  const parentNameId = getParentId(pathname, config)
-  if (!parentNameId || pages[parentNameId]) {
-    // parent already in the collection
+const directoryCollectionLoader = (
+  pathname,
+  options,
+  pages,
+  config,
+  buildFlags,
+) => {
+  const parentPath = path.dirname(pathname)
+  const parentId = computeParentId(pathname, config)
+  if (!parentId) {
+    // reached the top parent
+    return pages
+  }
+  if (
+    pages[parentId] &&
+    pages[parentId]._meta.buildVersion === buildFlags.version
+  ) {
+    // parent already in the collection and versions match
     return pages
   }
   let isTopLevel = true
-  const parentBasename = path.basename(parentName)
-  if (parentBasename !== parentName) {
+  const parentBasename = path.basename(parentPath)
+  if (parentBasename !== parentPath) {
     // there is a parent of the parent folder
     // according to our top down data cascade approach,
     // we need to compute this one first
     isTopLevel = false
-    pages = directoryCollectionLoader(parentName, options, pages, config)
+    pages = directoryCollectionLoader(
+      parentPath,
+      options,
+      pages,
+      config,
+      buildFlags,
+    )
   }
   let parent = baseLoader(
-    parentName,
-    { isDirectory: true, collectionGroup: "directory" },
+    parentPath,
+    {
+      isDirectory: true,
+      collectionGroup: "directory",
+      buildVersion: buildFlags.version,
+    },
     isTopLevel ? config.defaults.pageData : {},
     pages,
     config,
   )
 
   return _.merge({}, pages, {
-    [parentNameId]: parent,
+    [parentId]: parent,
   })
 }
 
@@ -343,106 +567,176 @@ const isComputableValue = (value) =>
   typeof value === "function" ||
   (_.isPlainObject(value) && value._kissCheckDependencies)
 
+const getFiles = (
+  loaderId,
+  config,
+  match = null,
+  contentPathInMatch = false,
+) => {
+  const loader = config.loaders[loaderId]
+  if (!loader) {
+    global.logger.error(`Loader with id ${loaderId} not found.`)
+    return []
+  }
+  // eslint-disable-next-line no-unused-vars
+  const { handler, namespace, ...loaderOptions } = loader
+  const allOptions = getOptions(config, namespace, loaderOptions)
+  const { matchOptions = {} } = allOptions
+  if (typeof match === "string") {
+    match = [match]
+  } else if (!match) {
+    match = allOptions.match
+  }
+  const fgOptions = {
+    ...matchOptions,
+    markDirectories: true,
+    stats: true,
+  }
+  if (!contentPathInMatch) {
+    fgOptions.cwd = config.dirs.content
+  }
+  return fg.sync(match, fgOptions).map((file) => ({
+    ...file,
+    loaderId,
+    // not sure why but it looks like sometimes fg returns the full path in the name
+    name: file.name.includes(path.sep) ? path.basename(file.name) : file.name,
+    // make sure path always includes the content directory
+    path: contentPathInMatch
+      ? file.path
+      : path.join(config.dirs.content, file.path),
+  }))
+}
+
 const getOptions = (config, namespace, options) => {
   const nameSpaceOptions = _.get(config, namespace, {})
   return { ...nameSpaceOptions, ...options }
 }
 
-const loadContent = async (config, context) => {
-  let pages = {}
+const loadContent = async (config, context, buildFlags) => {
+  const { incremental, buildPageIds } = buildFlags
+  let pages = context.pages
   let files = []
-  // We first need to fetch all files that match all loaders
-  // So that we can sort them in the right order before loading them
-  // This is essential for the data cascade to work properly
-  config.loaders.forEach(
-    ({ handler, namespace, source, ...loaderOptions }, idx) => {
-      if (source && source !== "file") {
-        return
-      }
-      const allOptions = getOptions(config, namespace, loaderOptions)
-      const { match, matchOptions = {}, ...options } = allOptions
-      if ("active" in options && !options.active) {
-        global.logger.log(`[${handler.name}]: loader not active. Skipping.`)
+  const isIncrementalBuild = incremental && buildPageIds?.length > 0
+
+  if (isIncrementalBuild) {
+    // incremental build: one content file changed
+    // -> reload the file + all their ascendants + any descendant
+    // files here are relative to the content directory
+
+    buildPageIds.forEach((id) => {
+      const page = pages[id]
+      if (!page) {
+        global.logger.warn(
+          `[loadContent] Page with id '${id}' not found. Skipping.`,
+        )
         return
       }
-      let fgOptions = {
-        cwd: config.dirs.content,
-        markDirectories: true,
-        stats: true,
-        ...matchOptions,
-      }
-      let extraOptions = []
-      if (namespace !== handler.name) {
-        extraOptions.push(`namespace: '${namespace}'`)
-      }
-      if (loaderOptions && Object.keys(loaderOptions).length > 0) {
-        extraOptions.push(`options: ${JSON.stringify(loaderOptions)}`)
+      // add to files to load
+      if (page._meta.indexInputPath) {
+        // also need to reload the index file
+        files = files.concat(
+          getFiles(
+            page._meta.indexLoaderId,
+            config,
+            page._meta.indexInputPath,
+            true,
+          ),
+        )
       }
-      global.logger.info(
-        `Listing files matching ${JSON.stringify(match)} for ${handler.name}${extraOptions.length > 0 ? ` (${extraOptions.join(", ")})` : ""}`,
-      )
       files = files.concat(
-        fg.sync(match, fgOptions).map((file) => ({ ...file, loaderIdx: idx })),
+        getFiles(page._meta.loaderId, config, page._meta.inputPath, true),
       )
-    },
-  )
-  global.logger.info(
-    `Found ${files.length} files. Sorting them for the data cascade.`,
-  )
+    })
+
+    global.logger.info(`Reloading ${files.length} files...`)
+  } else {
+    // We first need to fetch all files
+    // So that we can sort them in the right order before loading them
+    // This is essential for the data cascade to work properly
+    config.loaders.forEach(
+      ({ handler, namespace, source, ...loaderOptions }, idx) => {
+        if (source && source !== "file") {
+          return
+        }
+        const options = getOptions(config, namespace, loaderOptions)
+        if ("active" in options && !options.active) {
+          global.logger.log(`[${handler.name}]: loader not active. Skipping.`)
+          return
+        }
+        let extraOptions = []
+        if (namespace !== handler.name) {
+          extraOptions.push(`namespace: '${namespace}'`)
+        }
+        if (loaderOptions && Object.keys(loaderOptions).length > 0) {
+          extraOptions.push(`options: ${JSON.stringify(loaderOptions)}`)
+        }
+        // full build: load all files matching this loader
+        global.logger.info(
+          `Listing files matching ${JSON.stringify(options.match)} for ${handler.name}${extraOptions.length > 0 ? ` (${extraOptions.join(", ")})` : ""}`,
+        )
+        files = files.concat(getFiles(idx, config))
+      },
+    )
+    global.logger.info(`Found ${files.length} files. Loading...`)
+  }
   // sorting and loading files
   // sort files to make sure index.* files are loaded first and post.* files last
   // in order to respect the data cascade principle
   files = sortFiles(files)
-  global.logger.info(`Loading files...`)
   for (const file of files) {
     // finding the right loader for the file
     const { handler, namespace, ...loaderOptions } =
-      config.loaders[file.loaderIdx]
+      config.loaders[file.loaderId]
     const options = getOptions(config, namespace, loaderOptions)
-    let pathname = path.join(config.dirs.content, file.path)
     let page = {}
     try {
-      // load parent folders, if any
-      pages = directoryCollectionLoader(pathname, options, pages, config)
-      // load stats
-      page = {
-        _meta: {
-          fileCreated: file.stats.ctime,
-          fileModified: file.stats.mtime,
-        },
-      }
+      // initialize pages from directory structures
+      // this is so that we have a page entry from each folder, even if there are no index or post files
+      pages = directoryCollectionLoader(
+        file.path,
+        options,
+        pages,
+        config,
+        buildFlags,
+      )
       // load base data including _meta infos and based on ParentData
-      page = baseLoader(pathname, options, page, pages, config)
+      page = baseLoader(
+        file.path,
+        { ...options, file, buildVersion: buildFlags.version },
+        page,
+        pages,
+        config,
+      )
       // load content specific data
-      page = await handler(pathname, options, page, context, config)
+      page = await handler(file.path, options, page, context, config)
       // relative @attributes to absolute
       page = relativeToAbsoluteAttributes(page, options, config)
       pages[page._meta.id] = page
-      global.logger.log(`- [${handler.name}] loaded '${page._meta.inputPath}'`)
+      global.logger.log(`- [${handler.name}] loaded '${file.path}'`)
     } catch (err) {
       global.logger.error(
-        `- [${handler.name}] Error loading '${_.get(
-          page,
-          "_meta.inputPath",
-        )}'\n`,
+        `- [${handler.name}] Error loading '${file.path}'\n`,
         err.stack,
       )
     }
   }
-  // computed loaders
-  _.filter(config.loaders, (loader) => loader.source === "computed").forEach(
-    (loader) => {
-      const { description, handler, ...options } = loader
-      const message =
-        description || `Loading computed pages using ${handler.name}`
-      global.logger.info(message)
-      pages = handler(pages, options, config)
-    },
-  )
+  if (!incremental || buildPageIds?.length === 0) {
+    // full build
+    // computed loaders
+    _.filter(config.loaders, (loader) => loader.source === "computed").forEach(
+      (loader) => {
+        const { description, handler, ...options } = loader
+        const message =
+          description || `Loading computed pages using ${handler.name}`
+        global.logger.info(message)
+        pages = handler(pages, options, config)
+      },
+    )
+  }
   return pages
 }
 
-const runConfigHooks = (config, event, data) => {
+const runConfigHooks = (config, event, data, buildFlags) => {
   const hooks = _.get(config.hooks, event)
   if (!hooks || hooks.length === 0) {
     global.logger.log(`No hooks registered for ${event}`)
@@ -450,9 +744,19 @@ const runConfigHooks = (config, event, data) => {
   }
   hooks.forEach((hook) => {
     if (typeof hook === "function") {
-      data = runHandlerHook(hook, {}, config, data)
+      data = runHandlerHook(hook, {}, config, data, buildFlags)
     } else {
-      const { action, handler, command, ...options } = hook
+      const { action, handler, command, incrementalRebuild, ...options } = hook
+      if (buildFlags.incremental && buildFlags.file) {
+        // during an incremental rebuild, only run hooks that have an incrementalRebuild function set
+        if (
+          !incrementalRebuild ||
+          (typeof incrementalRebuild === "function" &&
+            !incrementalRebuild(buildFlags.file, data))
+        ) {
+          return
+        }
+      }
       switch (action) {
         case "copy":
           runCopyHook(options, config)
@@ -461,7 +765,7 @@ const runConfigHooks = (config, event, data) => {
           runExecHook(command, options)
           break
         case "run":
-          data = runHandlerHook(handler, options, config, data)
+          data = runHandlerHook(handler, options, config, data, buildFlags)
           break
         default: {
           global.logger.error(`Unknown hook action: ${action}`)
@@ -534,21 +838,23 @@ const sortFiles = (files) => {
   })
 }
 
-const runHandlerHook = (handler, options, config, data) => {
+const runHandlerHook = (handler, options, config, data, buildFlags) => {
   const message = options.description || `Running ${handler.name}`
   global.logger.info(message)
   try {
-    return handler(options, config, data)
+    return handler(options, config, data, buildFlags)
   } catch (err) {
     global.logger.error(`Error in ${handler.name}:\n`, err.stack)
     return data
   }
 }
 
-const writeStaticSite = async (context, config) => {
-  global.logger.info("Writing individual pages and images")
+const writeStaticSite = async (context, config, buildFlags) => {
+  const entries = getBuildEntries(context, buildFlags)
+  let message = `Writing ${entries.length} pages and images`
+  global.logger.info(message)
   await Promise.all(
-    _.map(context.pages, async (page) => {
+    entries.map(async ([, page]) => {
       if (page.excludeFromWrite) {
         global.logger.log(
           `- Page '${page._meta.id}' is marked as excludeFromWrite. Skipping.`,
@@ -557,6 +863,8 @@ const writeStaticSite = async (context, config) => {
         global.logger.log(
           `- Page '${page._meta.id}' has no permalink. Skipping.`,
         )
+      } else if (page._meta.outputType === "SKIP") {
+        global.logger.log(`- Page '${page._meta.id}' has SKIP type. Skipping.`)
       } else {
         const writer = config.writers.find(
           (writer) => writer.outputType === page._meta.outputType,

package/src/cli.js

@@ -27,4 +27,9 @@ yargs(hideBin(process.argv))
     default: false,
     describe: "Won't exit(1) on build errors",
   })
+  .option("incremental", {
+    boolean: true,
+    default: false,
+    describe: "Performs incremental builds on watch (experimental)",
+  })
   .demandCommand(1, "Enter a command").argv

package/src/data/computePermalink.js

@@ -36,7 +36,7 @@ const computePermalink = ({ slug, _meta }, config, { pages }) => {
     const parent = pages[_meta.parent]
     if (typeof parent.permalink === "function") {
       // return the function itself hoping parent permalink will be computed later
-      return computePermalink({ slug, _meta }, config, { pages })
+      return computePermalink
     }
     if (typeof parent.permalink === "string") {
       basePermalink = parent.permalink

package/src/data/initialPageData.js

@@ -45,10 +45,13 @@ const initialPageData = {
     descendants: computeDescendants,
     fileCreated: null,
     fileModified: null,
+    indexInputPath: null,
+    indexLoaderId: null,
     inputPath: "",
     isCollection: computeIsCollection,
     isDirectory: false,
     isPost: computeIsPost,
+    loaderId: null,
     outputPath: computeOutputPath,
     parent: null,
     outputType: null,

package/src/devServer.js

@@ -9,6 +9,11 @@ const { resetGlobalLogger } = require("./logger")
 
 const build = require("./build.js")
 
+let lastBuild = { context: null, config: null }
+let buildVersion = 0
+let isBuildRunning = false
+let changeBacklog = []
+
 const serve = (options, config) => {
   if (!config) {
     config = loadConfig(options)
@@ -48,14 +53,41 @@ const watch = async (options = {}, config) => {
       config.configFile,
       ...config.dirs.watchExtra,
     ],
-    { awaitWriteFinish: true },
+    { awaitWriteFinish: true, ignoreInitial: true },
   )
 
-  // rebuild on file changes
-  fileWatcher.on(
-    "all",
-    _.debounce((event, file) => rebuild(options, event, file), 500),
-  )
+  // initial build
+  lastBuild = await build(options)
+
+  if (options.incremental) {
+    console.info("Incremental build mode enabled")
+    // rebuild only the changed file
+    fileWatcher.on("all", async (event, file) => {
+      if (isBuildRunning) {
+        // there is already a build running, add to backlog
+        changeBacklog.push({ event, file })
+        return
+      }
+      isBuildRunning = true
+      lastBuild = await incrementalRebuild(options, lastBuild, event, file)
+      while (changeBacklog.length > 0) {
+        const change = changeBacklog.shift()
+        lastBuild = await incrementalRebuild(
+          options,
+          lastBuild,
+          change.event,
+          change.file,
+        )
+      }
+      isBuildRunning = false
+    })
+  } else {
+    // rebuild all on file changes
+    fileWatcher.on(
+      "all",
+      _.debounce((event, file) => rebuild(options, event, file), 500),
+    )
+  }
 
   // remove chokidar watcher
   let signals = [
@@ -71,7 +103,10 @@ const watch = async (options = {}, config) => {
   signals.forEach((signal) => {
     process.on(signal, (code) => {
       fileWatcher.close()
-      process.exit(code)
+      if (signal === "uncaughtException") {
+        global.logger.error("kiss encountered a fatal error\n", code)
+      }
+      process.exit(1)
     })
   })
 }
@@ -107,8 +142,18 @@ const preparePublicFolder = async (config) => {
 }
 
 const rebuild = (options, event, file) => {
-  console.info(`\nChange detected: [${event}] ${file}`)
+  global.logger.section(`\nChange detected: [${event}] ${file}`)
+  clearRequireCache()
+  resetGlobalLogger(options.verbosity)
+  return build(options)
+}
+
+const incrementalRebuild = async (options, lastBuild, event, file) => {
+  buildVersion++
+  global.logger.section(
+    `\nChange detected: [${event}] ${file}. Incremental rebuild #${buildVersion}...`,
+  )
   clearRequireCache()
   resetGlobalLogger(options.verbosity)
-  build(options)
+  return build({ event, file, ...options }, lastBuild, buildVersion)
 }

package/src/helpers.js

@@ -6,9 +6,28 @@ const path = require("path")
 // terminators: space, comma, newline, end of string, ', ",<, >, ), ], }, #
 const AT_GENERIC_ATTRIBUTE_REGEX =
   /@([a-zA-Z0-9-_]+):([^,\s\n\]'"<>)}#]+)(?=[,\s\n\]'"<>)}#]|$)/g
+// TODO: try with const AT_GENERIC_ATTRIBUTE_REGEX = /@([a-zA-Z0-9_-]+):([^,\s\]'"<>)}#]+)(?=[,\s\]'"<>)}#]|$)/g;
 
 const AT_FILE_ATTRIBUTE_REGEX =
   /@file:([^,\s\n\]'"<>)}#]+)(?=[,\s\n\]'"<>)}#]|$)/g
+// TODO: try with const AT_FILE_ATTRIBUTE_REGEX = /@file:([^\s,\\\]'"<>)}#]+)(?=[\s,\\\]'"<>)}#]|$)/g;
+
+const computePageId = (inputPath, config) => {
+  let topDir = config.dirs.content
+  if (config.dirs.content.endsWith("/")) {
+    topDir = topDir.slice(0, -1)
+  }
+  // replace top dir name by "."
+  return inputPath.replace(new RegExp("^" + topDir), ".")
+}
+
+const computeParentId = (inputPath, config) => {
+  const parentPath = path.dirname(inputPath)
+  if (parentPath === ".") {
+    return null
+  }
+  return computePageId(parentPath, config)
+}
 
 const findCollectionById = (collections, id) => {
   if (id === ".") {
@@ -51,6 +70,15 @@ const getAbsoluteURL = (url = "", baseURL) => {
   return new URL(url, baseURL).href
 }
 
+const getBuildEntries = (context, buildFlags) => {
+  const { pages } = context
+  const { buildPageIds } = buildFlags
+  if (buildPageIds?.length > 0) {
+    return buildPageIds.map((id) => [id, pages[id]])
+  }
+  return Object.entries(pages)
+}
+
 const getChildrenPages = (page, pages, filterOptions) => {
   const children = page._meta.children.map((c) => pages[c])
   if (filterOptions) {
@@ -138,6 +166,15 @@ const getLocale = (context, sep = "-") => {
   return ""
 }
 
+const getPageFromInputPath = (inputPath, pages) => {
+  const pageValues = Object.values(pages)
+  let page = pageValues.find(
+    (p) =>
+      p._meta.inputPath === inputPath || p._meta.indexInputPath === inputPath,
+  )
+  return page
+}
+
 // Tries to find the page corresponding to the source
 // @attributes should have been resolved by mow
 // Supports absolute, and relative paths
@@ -149,7 +186,13 @@ const getPageFromSource = (source, parentPage, pages, config, options = {}) => {
   const permalink = getFullPath(source, parentPage.permalink, {
     throwIfInvalid: true,
   })
-  const page = Object.values(pages).find((p) => p.permalink === permalink)
+  const page = Object.values(pages).find(
+    (p) =>
+      p.permalink === permalink ||
+      // in incremental builds,
+      // also search in derivatives in case the image source was already replaced during previous build
+      p.derivatives?.find((d) => d.permalink === permalink),
+  )
   if (!page) {
     if (throwIfNotFound) {
       throw new Error(
@@ -161,29 +204,17 @@ const getPageFromSource = (source, parentPage, pages, config, options = {}) => {
   return { ...page }
 }
 
-const getPageId = (inputPath, config) => {
-  let topDir = config.dirs.content
-  if (config.dirs.content.endsWith("/")) {
-    topDir = topDir.slice(0, -1)
-  }
-  // replace top dir name by "."
-  return inputPath.replace(new RegExp("^" + topDir), ".")
-}
-
-const getParentId = (inputPath, config) => {
-  const parentPath = path.dirname(inputPath)
-  if (parentPath === ".") {
-    return null
-  }
-  return getPageId(parentPath, config)
-}
-
 // returns the parent sanitizing the data for the cascade
-const getParentPage = (pages, id) => {
+const getParentPage = (pages, id, isPostAsking) => {
   let parent = pages[id]
   if (!parent) {
     global.logger.error(`Couldn't find parent with id '${id}'`)
-    console.log(Object.keys(pages))
+    return null
+  }
+  if (!isPostAsking) {
+    // omit attributes that shouldn't cascade, unless the page asking is a post
+    const attributesToOmit = ["_meta.indexInputPath", "_meta.indexLoaderId"]
+    parent = _.omit(parent, attributesToOmit)
   }
   return omitNoCascadeAttributes(parent)
 }
@@ -310,16 +341,18 @@ const sortPages = (pages, sortBy, { skipUndefinedSort } = {}) => {
 module.exports = {
   AT_FILE_ATTRIBUTE_REGEX,
   AT_GENERIC_ATTRIBUTE_REGEX,
+  computePageId,
+  computeParentId,
   findCollectionById,
   getAbsoluteURL,
+  getBuildEntries,
   getChildrenPages,
   getDescendantPages,
   getFullPath,
   getInputPath,
   getLocale,
+  getPageFromInputPath,
   getPageFromSource,
-  getPageId,
-  getParentId,
   getParentPage,
   isChild,
   isValidURL,
@@ -333,11 +366,11 @@ module.exports = {
 
 // removing any key ending with _no_cascade
 const omitNoCascadeAttributes = (obj) => {
-  let result = {}
+  let result = _.isArray(obj) ? [] : {}
   _.forEach(obj, (value, key) => {
-    if (_.isPlainObject(value)) {
+    if (_.isPlainObject(value) || _.isArray(value)) {
       result[key] = omitNoCascadeAttributes(value)
-    } else if (!key.endsWith("_no_cascade")) {
+    } else if (typeof key === "number" || !key.endsWith("_no_cascade")) {
       result[key] = value
     }
   })

package/src/libs/loadNunjucks.js

@@ -1,8 +1,13 @@
 const nunjucks = require("nunjucks")
 
-const loadNunjucks = (_, config) => {
+const loadNunjucks = (_, config, data, buildFlags = {}) => {
+  const options = {
+    // we need to watch them so that template can be reloaded when in incremental mode
+    // (in normal mode, config is reloaded which reloads the lib and clears the cache)
+    watch: !!buildFlags.incremental,
+  }
   // set default nunjucks template dir
-  config.libs.nunjucks = nunjucks.configure(config.dirs.template)
+  config.libs.nunjucks = nunjucks.configure(config.dirs.template, options)
   // clear nunjucks caches (for when in watch mode)
   // https://risanb.com/code/how-to-clear-nunjucks-cache/
   config.libs.nunjucks.loaders.map((loader) => (loader.cache = {}))

package/src/loaders/baseLoader.js

@@ -1,19 +1,23 @@
 const path = require("path")
 const _ = require("lodash")
 
-const { getPageId, getParentId, getParentPage } = require("../helpers")
+const { computePageId, computeParentId, getParentPage } = require("../helpers")
 
-const baseLoader = (inputPath, options, page, pages, config) => {
-  let parentId = getParentId(inputPath, config)
-  let basename = path.basename(inputPath)
+const baseLoader = (inputPath, options = {}, page = {}, pages, config) => {
+  const { file, buildVersion } = options
+  const inputPathObject = path.parse(inputPath)
+  let parentId = computeParentId(inputPath, config)
+  let basename = inputPathObject.base
   if (basename === config.dirs.content) {
     basename = ""
   }
-  const parentData = parentId ? getParentPage(pages, parentId) : {}
+  const parentData = parentId
+    ? getParentPage(pages, parentId, inputPathObject.name === "post")
+    : {}
   let isDirectory = options.isDirectory || inputPath.endsWith("/")
-  let id = options.id || getPageId(inputPath, config)
+  let id = options.id || computePageId(inputPath, config)
   let outputType = options.outputType || "HTML"
-  let collectionGroup = options.collectionGroup
+  let collectionGroup = options.collectionGroup || "directory"
   if (outputType === "HTML") {
     // only files converted as HTML can override
     // they parent entry
@@ -48,6 +52,20 @@ const baseLoader = (inputPath, options, page, pages, config) => {
     outputType,
     collectionGroup,
   })
+  if (inputPathObject.name === "index") {
+    // save the input path in indexInputPath in case it gets overwritten later on by a post.* file
+    page._meta.indexInputPath = inputPath
+    // also need to save the loader index
+    page._meta.indexLoaderId = file?.loaderId
+  }
+  if (file) {
+    page._meta.fileCreated = file.stats.ctime
+    page._meta.fileModified = file.stats.mtime
+    page._meta.loaderId = file.loaderId
+  }
+  if (buildVersion !== undefined) {
+    page._meta.buildVersion = buildVersion
+  }
   return page
 }
 

package/src/loaders/jsLoader.js

@@ -1,7 +1,7 @@
 const path = require("path")
 
-const jsLoader = (id, options, page) => {
-  const filePath = path.resolve(page._meta.inputPath)
+const jsLoader = (inputPath, options, page) => {
+  const filePath = path.resolve(inputPath)
   const fileData = require(filePath)
   return { ...page, ...fileData }
 }

package/src/loaders/jsonLoader.js

@@ -1,8 +1,8 @@
 const { parseISO } = require("date-fns")
 const { readJsonSync } = require("fs-extra")
 
-const jsonLoader = (path, options, page, pages, config) => {
-  let fileData = readJsonSync(path)
+const jsonLoader = (inputPath, options, page, pages, config) => {
+  let fileData = readJsonSync(inputPath)
   const published = fileData[config.defaults.pagePublishedAttribute]
   const updated = fileData[config.defaults.pageUpdatedAttribute]
   if (published && typeof published === "string") {

package/src/loaders/markdownLoader.js

@@ -1,7 +1,7 @@
 const textLoader = require("./textLoader")
 
-const markdownLoader = (id, options, page, _, config) => {
-  const fileData = textLoader(id, options, page)
+const markdownLoader = (inputPath, options, page, _, config) => {
+  const fileData = textLoader(inputPath, options, page)
   const content = config.libs.marked(fileData.content)
   return { ...page, ...fileData, content }
 }

package/src/loaders/staticLoader.js

@@ -1,6 +1,6 @@
 const _ = require("lodash")
 
-const staticLoader = (id, options, { permalink, _meta }) => {
+const staticLoader = (inputPath, options, { permalink, _meta }) => {
   return {
     permalink,
     _meta: {

package/src/loaders/textLoader.js

@@ -1,8 +1,8 @@
 const frontMatter = require("front-matter")
 const { readFileSync } = require("fs-extra")
 
-const textLoader = (id, _, page) => {
-  const fileContent = readFileSync(page._meta.inputPath, "utf8")
+const textLoader = (inputPath, _, page) => {
+  const fileContent = readFileSync(inputPath, "utf8")
   const fileData = frontMatter(fileContent)
   const content = fileData.body
   return { ...page, ...fileData.attributes, content }

package/src/transforms/imageContextTransform.js

@@ -3,7 +3,7 @@ const cheerio = require("cheerio")
 const path = require("path")
 const sharp = require("sharp")
 
-const { getPageFromSource, isValidURL } = require("../helpers")
+const { getBuildEntries, getPageFromSource, isValidURL } = require("../helpers")
 
 const META_SELECTORS = [
   "meta[property='og:image']",
@@ -11,8 +11,10 @@ const META_SELECTORS = [
   "meta[property='twitter:image']",
 ]
 
-const imageContextTransform = async (context, options, config) => {
-  for (let [id, page] of Object.entries(context.pages)) {
+const imageContextTransform = async (context, options, config, buildFlags) => {
+  const entries = getBuildEntries(context, buildFlags)
+
+  for (let [id, page] of entries) {
     if (!page._html || !page.permalink) {
       continue
     }
@@ -253,14 +255,16 @@ const transformImageTag = async ($, img, page, context, options, config) => {
     global.logger.warn(`Page '${id}': image '${src}' has no 'alt' attribute.`)
   }
   if (isValidURL(src)) {
-    global.logger.log(`Page '${id}': image '${src}' is a URL. Skipping.`)
+    global.logger.log(
+      `- [transformImageTag] Page '${id}': image '${src}' is a URL. Skipping.`,
+    )
     return context
   }
   const imgPage = getPageFromSource(src, page, context.pages, config)
   if (imgPage._meta.outputType !== "IMAGE") {
     // image is handled by another loader. skipping.
     global.logger.log(
-      `Page '${id}': image '${src}' is handled by another loader. Skipping.`,
+      `- [transformImageTag] Page '${id}': image '${src}' is handled by another loader. Skipping.`,
     )
     return context
   }
@@ -304,7 +308,7 @@ const transformMetaTag = async (
   if (imgPage._meta.outputType !== "IMAGE") {
     // image is handled by another loader. skipping.
     global.logger.log(
-      `Page '${id}': image '${url.pathname}' is handled by another loader. Skipping.`,
+      `- [transformMetaTag] Page '${id}': image '${url.pathname}' is handled by another loader. Skipping.`,
     )
     return context
   }

package/src/views/computeCollectionDataView.js

@@ -41,7 +41,11 @@ module.exports = computeCollectionDataView
 
 const getAllPosts = (page, pages, config) =>
   getDescendantPages(page, pages, {
-    filterBy: (p) => p._meta.isPost && p.permalink && !p.excludeFromCollection,
+    filterBy: (p) =>
+      p._meta.isPost &&
+      p.permalink &&
+      !p.excludeFromCollection &&
+      !p.excludeFromWrite,
     sortBy: page.sortCollectionBy || config.defaults.sortCollectionBy,
     skipUndefinedSort: true,
   })

package/src/writers/rssContextWriter.js

@@ -57,7 +57,7 @@ const pageObject = (page, options, config) => {
       publishedDate = formatRFC3339(publishedDate)
     } catch (err) {
       global.logger.warn(
-        `[${this.name}]: cannot format published date '${publishedDate}' to RFC3339 for page '${page.permalink}': ${err}.`,
+        `[rssContextWriter]: cannot format published date '${publishedDate}' to RFC3339 for page '${page.permalink}': ${err}.`,
       )
       publishedDate = null
     }
@@ -68,7 +68,7 @@ const pageObject = (page, options, config) => {
       updatedDate = formatRFC3339(updatedDate)
     } catch (err) {
       global.logger.warn(
-        `[${this.name}]: cannot format updated date '${updatedDate}' to RFC3339 for page '${page.permalink}': ${err}.`,
+        `[rssContextWriter]: cannot format updated date '${updatedDate}' to RFC3339 for page '${page.permalink}': ${err}.`,
       )
       updatedDate = null
     }