@mnstry/atelier 0.2.0-alpha.1 → 0.2.0-alpha.11

package/src/graph/graph.mjs

@@ -1,261 +1,182 @@
 import fs from 'node:fs'
 import path from 'node:path'
-import { commandProject, gitRemoteUrl, loadRepoAccess, readJson, remoteHost, writeJson } from '../project/config.mjs'
-import { gitIgnoreFilter } from '../project/git-ignore.mjs'
+import { commandProject, gitRemoteUrl, loadRepoAccess, remoteHost, writeJson } from '../project/config.mjs'
+import { VALID_AUDIENCES as PROJECTION_AUDIENCES } from '../projection/policy.mjs'
+import {
+  REPO_ACCESS_SCHEMA,
+  VALID_RELATION_TYPES,
+  buildKnowledgeGraph,
+  createGraphFileCache,
+  parseYamlSubset,
+} from './knowledge-graph.mjs'
+
+export { createGraphFileCache }
+
+// Compatibility exports. Parsing, census, validation, and diagnostics all live
+// in knowledge-graph.mjs; this module only projects its canonical result into
+// the long-standing `atelier graph` artifact shape.
+export const VALID_AUDIENCES = PROJECTION_AUDIENCES
+export const VALID_RELATIONS = VALID_RELATION_TYPES
+export const parseFrontmatterYaml = parseYamlSubset
+
+function canonicalInput(project) {
+  const loadedAccess = loadRepoAccess(project)
+  const managed = []
+  const missing = []
+  const accessRepos = {}
 
-export const VALID_AUDIENCES = new Set(['private', 'team', 'operator', 'staff', 'public', 'sensitive'])
-export const VALID_RELATIONS = new Set(['related', 'supports', 'supersedes', 'implements', 'depends_on', 'evidences', 'contradicts', 'belongs_to'])
-const DOC_EXTS = new Set(['.md', '.html', '.pdf', '.docx'])
-const SKIP_DIRS = new Set(['.git', 'node_modules', 'atelier-output', 'atelier-readers', '.mnstry', '.atelier-local'])
-
-const slug = (value) => String(value ?? '').toLowerCase().replace(/\.[^.]+$/, '').replace(/[^a-z0-9]+/g, '-').replace(/^-+|-+$/g, '')
-const relPath = (root, file) => path.relative(root, file).split(path.sep).join('/')
-
-// Sidecar-first census. Document extensions are always sources; any other
-// file — json, yaml, csv, binary assets — opts in by carrying an adjacent
-// .kg.json sidecar. The sidecar itself is metadata, never a source asset.
-//
-// Membership asks whether the sidecar is VISIBLE, not whether it happens to
-// exist on this disk. A git-ignored sidecar is machine-local: obeying one
-// enrolls a node that no tracked file declares — audience and all — so a
-// clean checkout builds a different graph than the machine that wrote it.
-function isSourceFile(name, sidecarVisible) {
-  if (name.endsWith('.kg.json')) return false
-  if (DOC_EXTS.has(path.extname(name).toLowerCase())) return true
-  return sidecarVisible === true
-}
-
-function sidecarIsVisible(abs, root, isIgnored) {
-  const sidecar = sidecarFor(abs)
-  return fs.existsSync(sidecar) && !isIgnored(relPath(root, sidecar))
-}
-
-function walk(dir, root, acc = [], isIgnored = gitIgnoreFilter(root)) {
-  for (const ent of fs.readdirSync(dir, { withFileTypes: true })) {
-    if (ent.name.startsWith('.') || SKIP_DIRS.has(ent.name)) continue
-    const abs = path.join(dir, ent.name)
-    if (isIgnored(relPath(root, abs))) continue
-    if (ent.isDirectory()) walk(abs, root, acc, isIgnored)
-    if (!ent.isFile() || ent.name.endsWith('.kg.json')) continue
-    const ext = path.extname(ent.name).toLowerCase()
-    // Markdown carries its own front matter and never consults a sidecar.
-    const sidecarVisible = ext === '.md' ? false : sidecarIsVisible(abs, root, isIgnored)
-    if (isSourceFile(ent.name, sidecarVisible)) acc.push({ abs, rel: relPath(root, abs), ext, sidecarVisible })
-  }
-  return acc
-}
-
-function splitFrontmatter(raw) {
-  if (!raw.startsWith('---\n')) return null
-  const end = raw.indexOf('\n---', 4)
-  if (end === -1) return null
-  return raw.slice(4, end)
-}
-
-function parseValue(value) {
-  const text = String(value ?? '').trim()
-  if (!text) return ''
-  if ((text.startsWith('"') && text.endsWith('"')) || (text.startsWith("'") && text.endsWith("'"))) return text.slice(1, -1)
-  if (text === 'true') return true
-  if (text === 'false') return false
-  return text
-}
-
-function setPath(target, keys, value) {
-  let cursor = target
-  for (const key of keys.slice(0, -1)) {
-    cursor[key] ??= {}
-    cursor = cursor[key]
-  }
-  cursor[keys.at(-1)] = value
-}
-
-export function parseFrontmatterYaml(yaml) {
-  const out = {}
-  const lines = String(yaml ?? '').split('\n')
-  const stack = [{ indent: -1, value: out, key: null }]
-  for (let index = 0; index < lines.length; index += 1) {
-    const raw = lines[index]
-    if (!raw.trim() || raw.trim().startsWith('#')) continue
-    const indent = raw.match(/^\s*/)[0].length
-    const line = raw.trim()
-    while (stack.length > 1 && indent <= stack.at(-1).indent) stack.pop()
-    const parent = stack.at(-1).value
-    if (line.startsWith('- ')) {
-      if (!Array.isArray(parent)) continue
-      parent.push(parseValue(line.slice(2)))
+  for (const repo of project.repos ?? []) {
+    if (repo.external) continue
+    if (!repo.name || !repo.path || !fs.existsSync(repo.path)) {
+      missing.push(`configured repo is missing or unreadable: ${repo.name ?? '(unnamed)'}`)
       continue
     }
-    const match = line.match(/^([A-Za-z0-9_.-]+):\s*(.*)$/)
-    if (!match) continue
-    const [, key, rest] = match
-    if (rest === '') {
-      parent[key] = nextYamlContainer(lines, index, indent)
-      stack.push({ indent, value: parent[key], key })
-    } else if (rest === '[]') {
-      parent[key] = []
-      stack.push({ indent, value: parent[key], key })
-    } else {
-      parent[key] = parseValue(rest)
+    managed.push({ name: repo.name, path: repo.path })
+    accessRepos[repo.name] = {
+      readBoundary:
+        loadedAccess.repos?.[repo.name]?.readBoundary ?? repo.readBoundary ?? loadedAccess.defaultReadBoundary ?? 'team',
     }
   }
-  return out
-}
-
-function nextYamlContainer(lines, fromIndex, indent) {
-  for (let index = fromIndex + 1; index < lines.length; index += 1) {
-    const raw = lines[index]
-    if (!raw.trim() || raw.trim().startsWith('#')) continue
-    const nextIndent = raw.match(/^\s*/)[0].length
-    if (nextIndent <= indent) return {}
-    return raw.trim().startsWith('- ') ? [] : {}
-  }
-  return {}
-}
-
-function normalizeRelations(relations) {
-  const out = {}
-  if (!relations || typeof relations !== 'object') return out
-  for (const [type, targets] of Object.entries(relations)) {
-    if (Array.isArray(targets)) out[type] = targets.map(String)
-    else if (typeof targets === 'string') out[type] = [targets]
-    else out[type] = []
-  }
-  return out
-}
-
-function sidecarFor(file) {
-  return `${file}.kg.json`
-}
-
-function validateSidecar(meta, repo, file, errors) {
-  const allowed = new Set(['assetFilename', 'title', 'summary', 'tags', 'kg'])
-  const kgAllowed = new Set(['id', 'title', 'type', 'domain', 'lifecycle', 'status', 'audience', 'relations'])
-  for (const key of Object.keys(meta || {})) {
-    if (!allowed.has(key)) errors.push(`${repo.name}/${file.rel}: sidecar has unknown key ${key}`)
-  }
-  for (const key of Object.keys(meta?.kg || {})) {
-    if (!kgAllowed.has(key)) errors.push(`${repo.name}/${file.rel}: sidecar kg has unknown key ${key}`)
-  }
-}
 
-function nodeFromMarkdown(repo, file, errors) {
-  const raw = fs.readFileSync(file.abs, 'utf8')
-  const fm = splitFrontmatter(raw)
-  if (!fm) return null
-  const parsed = parseFrontmatterYaml(fm)
-  const kg = parsed.kg ?? {}
-  if (!kg.id) errors.push(`${repo.name}/${file.rel}: kg.id is required`)
-  if (!kg.audience) errors.push(`${repo.name}/${file.rel}: kg.audience is required`)
-  if (kg.visibility) errors.push(`${repo.name}/${file.rel}: kg.visibility is invalid; use kg.audience`)
   return {
-    id: kg.id || `${repo.name}:${slug(file.rel)}`,
-    title: parsed.title || kg.title || path.basename(file.rel, file.ext),
-    summary: parsed.summary || '',
-    repo: repo.name,
-    path: file.rel,
-    type: kg.type || 'document',
-    status: kg.status || 'active',
-    audience: kg.audience || 'private',
-    tags: Array.isArray(parsed.tags) ? parsed.tags : [],
-    relations: normalizeRelations(kg.relations),
-  }
-}
-
-function nodeFromSidecar(repo, file, errors) {
-  const sidecar = sidecarFor(file.abs)
-  // Visibility again, and for the same reason: a git-ignored sidecar must not
-  // supply identity, audience, or relations. A document-extension asset whose
-  // only sidecar is ignored therefore fails closed with the ordinary
-  // missing-sidecar error — the same verdict a clean checkout reaches.
-  if (!file.sidecarVisible) {
-    errors.push(`${repo.name}/${file.rel}: non-Markdown source requires sidecar ${path.basename(sidecar)}`)
-    return null
-  }
-  const meta = readJson(sidecar)
-  validateSidecar(meta, repo, file, errors)
-  const kg = meta.kg ?? {}
-  if (meta.assetFilename && meta.assetFilename !== path.basename(file.abs)) errors.push(`${repo.name}/${file.rel}: sidecar assetFilename must match source filename`)
-  if (!kg.id) errors.push(`${repo.name}/${file.rel}: sidecar kg.id is required`)
-  if (!kg.audience) errors.push(`${repo.name}/${file.rel}: sidecar kg.audience is required`)
-  if (kg.visibility) errors.push(`${repo.name}/${file.rel}: sidecar kg.visibility is invalid; use kg.audience`)
+    missing,
+    options: {
+      workspaceRoot: project.workspaceRoot,
+      repoEntries: managed,
+      repoAccessConfig: {
+        schema: REPO_ACCESS_SCHEMA,
+        defaultReadBoundary: loadedAccess.defaultReadBoundary ?? 'team',
+        repos: accessRepos,
+      },
+      repoAccessConfigPath: project.repoAccessPath,
+      externalRelationPrefixes: Array.isArray(project.config.graph?.externalRelationPrefixes)
+        ? project.config.graph.externalRelationPrefixes
+        : [],
+      externalRelationIds: Array.isArray(project.config.graph?.externalRelationIds) ? project.config.graph.externalRelationIds : [],
+    },
+  }
+}
+
+function compatibilityError(message) {
+  return String(message)
+    .replace(': missing non-Markdown sidecar ', ': non-Markdown source requires sidecar ')
+    .replace(/: active orphan sidecar has no adjacent source .+$/, ': sidecar has no matching source asset')
+}
+
+function compatibilityNode(node) {
+  return {
+    id: node.id,
+    title: node.title,
+    summary: node.summary,
+    repo: node.repo,
+    path: node.path,
+    type: node.kgType,
+    status: node.status,
+    audience: node.audience,
+    ...(node.classification === 'unclassified'
+      ? { classification: 'unclassified', classificationReason: node.classificationReason }
+      : {}),
+    tags: node.tags,
+    relations: node.relations,
+    repoAccess: node.repoAccess,
+  }
+}
+
+function compatibilityDiagnostic(diagnostic) {
+  if (diagnostic.code === 'unclassified-content') return diagnostic
+  return {
+    severity: diagnostic.severity,
+    code: 'audience-wider-repo-boundary',
+    node: diagnostic.node,
+    message: diagnostic.message,
+  }
+}
+
+function externalRepos(project) {
+  return (project.repos ?? [])
+    .filter((repo) => repo.external)
+    .map((repo) => ({
+      name: repo.name,
+      path: repo.path ?? null,
+      remoteHost: remoteHost(gitRemoteUrl(repo.path)),
+    }))
+    .sort((a, b) => String(a.name).localeCompare(String(b.name)))
+}
+
+function canonicalBuild(project, overrides = {}) {
+  const input = canonicalInput(project)
+  const result = buildKnowledgeGraph({ ...input.options, ...overrides })
+  return { input, result }
+}
+
+// The supported project-to-options seam: the exact buildKnowledgeGraph options
+// every graph consumer derives from a loaded project, plus the configured
+// repositories that could not be read.
+export function canonicalGraphOptions(project) {
+  return canonicalInput(project)
+}
+
+const canonicalEdgeId = (edge) => JSON.stringify([edge.source, edge.type, edge.target])
+
+// The canonical graph for a loaded project, for consumers that need derived
+// edges and the source offsets behind them. Nothing is resolved here: nodes,
+// edges, link occurrences and link findings all come from
+// buildKnowledgeGraph. Each edge gains a stable id and an origin; a derived
+// edge carries every source occurrence that produced it. Embedded assets are
+// reported beside the graph, like link occurrences: `embeds` holds one record
+// per embed occurrence and `assets` each embedded file once. Neither adds a
+// node or an edge.
+// `fileCache` (createGraphFileCache) lets repeated builds of one workspace
+// reuse the per-file census and link scan of every Markdown source whose
+// bytes did not change; the result is the same as without it. `observedDigest`
+// (see createGraphFileCache) lets a caller that observes sources by digest
+// spare the read of a source it knows unchanged.
+export function buildCanonicalGraph(project, { isLinkTargetEligible, isAssetEligible, fileCache = null, observedDigest = null } = {}) {
+  const { input, result } = canonicalBuild(project, {
+    ...(isLinkTargetEligible ? { isLinkTargetEligible } : {}),
+    ...(isAssetEligible ? { isAssetEligible } : {}),
+    ...(fileCache ? { fileCache } : {}),
+    ...(observedDigest ? { observedDigest } : {}),
+  })
+  const canonical = result.workspaceGraph ?? { nodes: [], edges: [], diagnostics: [] }
+  const occurrences = new Map()
+  for (const link of result.resolvedLinks ?? []) {
+    const id = canonicalEdgeId(link)
+    if (!occurrences.has(id)) occurrences.set(id, [])
+    occurrences.get(id).push(link)
+  }
+  const edges = canonical.edges.map((edge) => {
+    const id = canonicalEdgeId(edge)
+    return edge.declared === true
+      ? { id, ...edge, origin: 'declared' }
+      : { id, ...edge, origin: 'ordinary-link', occurrences: occurrences.get(id) ?? [] }
+  })
+  const embeds = result.resolvedEmbeds ?? []
+  const assets = [...new Map(embeds.map((embed) => [embed.asset.id, embed.asset])).values()].sort((a, b) => (a.id < b.id ? -1 : a.id > b.id ? 1 : 0))
   return {
-    id: kg.id || `${repo.name}:${slug(file.rel)}-${file.ext.slice(1)}`,
-    title: meta.title || kg.title || path.basename(file.rel, file.ext),
-    summary: meta.summary || '',
-    repo: repo.name,
-    path: file.rel,
-    type: kg.type || file.ext.slice(1),
-    status: kg.status || 'active',
-    audience: kg.audience || 'private',
-    tags: Array.isArray(meta.tags) ? meta.tags : [],
-    relations: normalizeRelations(kg.relations),
+    ok: result.ok && input.missing.length === 0,
+    errors: [...input.missing, ...result.errors],
+    options: input.options,
+    nodes: canonical.nodes,
+    edges,
+    diagnostics: canonical.diagnostics,
+    links: result.resolvedLinks ?? [],
+    embeds,
+    assets,
+    linkDiagnostics: result.linkDiagnostics ?? [],
+    ...(result.fileCensus ? { fileCensus: result.fileCensus } : {}),
   }
 }
 
 export function buildGraph(project) {
-  const repoAccess = loadRepoAccess(project)
-  const externalRelationPrefixes = new Set(Array.isArray(project.config.graph?.externalRelationPrefixes) ? project.config.graph.externalRelationPrefixes : [])
-  const externalRelationIds = new Set(Array.isArray(project.config.graph?.externalRelationIds) ? project.config.graph.externalRelationIds : [])
-  const errors = []
-  const diagnostics = []
-  const nodes = []
-  const edges = []
-  const external = []
-
-  for (const repo of project.repos) {
-    if (repo.external) {
-      // Identity acknowledged, nothing walked: the workspace does not own this
-      // content and cannot answer for its metadata.
-      external.push({ name: repo.name, path: repo.path ?? null, remoteHost: remoteHost(gitRemoteUrl(repo.path)) })
-      continue
-    }
-    if (!repo.name || !repo.path || !fs.existsSync(repo.path)) {
-      errors.push(`configured repo is missing or unreadable: ${repo.name ?? '(unnamed)'}`)
-      continue
-    }
-    // One batched ignore lookup per repo, shared by every walk below.
-    const isIgnored = gitIgnoreFilter(repo.path)
-    for (const file of walk(repo.path, repo.path, [], isIgnored)) {
-      // Markdown is the one inline metadata format; every other source is
-      // sidecar-described and its own bytes are never read.
-      const node = file.ext === '.md' ? nodeFromMarkdown(repo, file, errors) : nodeFromSidecar(repo, file, errors)
-      if (!node) continue
-      const boundary = repoAccess.repos?.[repo.name]?.readBoundary ?? repo.readBoundary ?? repoAccess.defaultReadBoundary ?? 'team'
-      node.repoAccess = { readBoundary: boundary }
-      if (['private', 'sensitive'].includes(node.audience) && boundary !== 'private') {
-        diagnostics.push({
-          severity: 'warning',
-          code: 'audience-wider-repo-boundary',
-          node: node.id,
-          message: `${node.id}: audience ${node.audience} is in a ${boundary}-readable repo`,
-        })
-      }
-      nodes.push(node)
-    }
-    for (const orphan of findOrphanSidecars(repo.path, isIgnored)) {
-      errors.push(`${repo.name}/${orphan}: sidecar has no matching source asset`)
-    }
-  }
-
-  const ids = new Set()
-  for (const node of nodes) {
-    if (ids.has(node.id)) errors.push(`duplicate kg.id: ${node.id}`)
-    ids.add(node.id)
-    if (!VALID_AUDIENCES.has(node.audience)) errors.push(`${node.id}: invalid audience ${node.audience}`)
-  }
-
-  for (const node of nodes) {
-    for (const [type, targets] of Object.entries(node.relations ?? {})) {
-      if (!VALID_RELATIONS.has(type)) errors.push(`${node.id}: invalid relation type ${type}`)
-      for (const target of targets) {
-        if (!ids.has(target) && !isExternalRelationTarget(target, { externalRelationPrefixes, externalRelationIds })) {
-          errors.push(`${node.id}: relation target not found: ${target}`)
-        }
-        edges.push({ source: node.id, target, type, declared: true })
-      }
-    }
-  }
+  const { input, result } = canonicalBuild(project)
+  const canonical = result.workspaceGraph ?? { nodes: [], edges: [], diagnostics: [] }
+  const nodes = canonical.nodes.map(compatibilityNode)
+  // The legacy artifact exposes declared relations only. Derived Markdown-link
+  // edges remain available from buildKnowledgeGraph without changing this API.
+  const edges = canonical.edges.filter((edge) => edge.declared === true)
+  const diagnostics = canonical.diagnostics.map(compatibilityDiagnostic)
+  const errors = [...input.missing, ...result.errors.map(compatibilityError)]
 
   return {
     schema: 'mnstry.atelier-knowledge-graph@v1',
@@ -265,83 +186,22 @@ export function buildGraph(project) {
     nodes,
     edges,
     diagnostics,
-    external: external.sort((a, b) => String(a.name).localeCompare(String(b.name))),
+    external: externalRepos(project),
     errors,
   }
 }
 
-function isExternalRelationTarget(target, { externalRelationPrefixes, externalRelationIds }) {
-  if (externalRelationIds.has(target)) return true
-  const prefix = String(target).split(':')[0]
-  return externalRelationPrefixes.has(prefix)
-}
-
-function findOrphanSidecars(root, isIgnored = gitIgnoreFilter(root)) {
-  const orphans = []
-  function visit(dir) {
-    for (const ent of fs.readdirSync(dir, { withFileTypes: true })) {
-      if (ent.name.startsWith('.') || SKIP_DIRS.has(ent.name)) continue
-      const abs = path.join(dir, ent.name)
-      if (isIgnored(relPath(root, abs))) continue
-      if (ent.isDirectory()) visit(abs)
-      else if (ent.isFile() && ent.name.endsWith('.kg.json')) {
-        const asset = abs.slice(0, -'.kg.json'.length)
-        if (!fs.existsSync(asset)) orphans.push(relPath(root, abs))
-      }
-    }
-  }
-  visit(root)
-  return orphans
-}
-
-// Sidecars that exist on disk but are git-ignored, next to a file the census
-// can see. Refused, never obeyed — and never silently: dropping them without a
-// word is what let an untracked sidecar mint a public node.
-function findIgnoredSidecars(root, isIgnored = gitIgnoreFilter(root)) {
-  const found = []
-  function visit(dir) {
-    for (const ent of fs.readdirSync(dir, { withFileTypes: true })) {
-      if (ent.name.startsWith('.') || SKIP_DIRS.has(ent.name)) continue
-      const abs = path.join(dir, ent.name)
-      const rel = relPath(root, abs)
-      if (ent.isDirectory()) {
-        if (!isIgnored(rel)) visit(abs)
-        continue
-      }
-      if (!ent.isFile() || !ent.name.endsWith('.kg.json') || !isIgnored(rel)) continue
-      const assetRel = rel.slice(0, -'.kg.json'.length)
-      // Nothing was refused if the asset is itself machine-local or absent, and
-      // Markdown never reads a sidecar in the first place.
-      if (assetRel.toLowerCase().endsWith('.md')) continue
-      if (isIgnored(assetRel) || !fs.existsSync(path.join(root, assetRel))) continue
-      found.push({ path: assetRel, sidecar: rel })
-    }
-  }
-  visit(root)
-  return found
-}
-
-// Deliberately NOT part of buildGraph's return value: that object is written
-// verbatim to the graph artifact, so a machine-local observation inside it
-// would make the artifact differ between a developer's tree and a clean
-// checkout — the exact churn the ignore filter exists to prevent. The operator
-// hears about the refusal; the shared artifact never does.
+// Machine-local ignored-sidecar observations stay outside generated artifacts.
 export function ignoredSidecarWarnings(project) {
-  const warnings = []
-  for (const repo of project.repos ?? []) {
-    if (repo.external || !repo.name || !repo.path || !fs.existsSync(repo.path)) continue
-    for (const found of findIgnoredSidecars(repo.path)) {
-      warnings.push({
-        severity: 'warning',
-        code: 'ignored-sidecar',
-        repo: repo.name,
-        path: found.path,
-        sidecar: found.sidecar,
-        message: `${repo.name}/${found.sidecar}: sidecar is git-ignored and was refused; it cannot enroll ${found.path} or describe it`,
-      })
-    }
-  }
-  return warnings
+  const { result } = canonicalBuild(project)
+  return (result.ignoredSidecars ?? []).map((found) => ({
+    severity: 'warning',
+    code: 'ignored-sidecar',
+    repo: found.repo,
+    path: found.path,
+    sidecar: found.sidecar,
+    message: `${found.repo}/${found.sidecar}: sidecar is git-ignored and was refused; it cannot enroll ${found.path} or describe it`,
+  }))
 }
 
 export function runGraphCommand(argv = process.argv.slice(2)) {