@gitroll/tacitt-cli 0.0.0-stage → 0.1.0

package/skills/tacitt/scripts/journey-classification-validation.mjs

@@ -0,0 +1,219 @@
+const types = ['work', 'side', 'coursework', 'personal', 'scratch', 'other']
+const access = ['public', 'private', 'drop']
+const text = (value, max) =>
+  typeof value === 'string' && value.trim().length > 0 && value.length <= max
+
+export function validateClassificationJudgment(value, key) {
+  if (
+    !value ||
+    value.rootKey !== key ||
+    !text(value.projectName, 240) ||
+    !text(value.workIntent, 240) ||
+    !text(value.groupingBasis, 200) ||
+    (value.targetProjectId !== undefined &&
+      (typeof value.targetProjectId !== 'string' || !value.targetProjectId)) ||
+    Object.keys(value).some(
+      (field) =>
+        !['rootKey', 'projectName', 'workIntent', 'groupingBasis', 'targetProjectId'].includes(
+          field,
+        ),
+    )
+  )
+    throw new Error('CLASSIFICATION_JUDGMENT_INVALID')
+}
+
+export function validateClassificationProject(value, projectId) {
+  if (
+    !value ||
+    value.projectId !== projectId ||
+    !text(value.description, 1000) ||
+    !types.includes(value.projectType) ||
+    !access.includes(value.recommendation?.access) ||
+    !text(value.recommendation?.reason, 200) ||
+    Object.keys(value).some(
+      (field) => !['projectId', 'description', 'projectType', 'recommendation'].includes(field),
+    )
+  )
+    throw new Error('PROJECT_METADATA_INVALID')
+}
+
+export function validateConsolidationAnswer(answer, descriptor) {
+  const details = []
+  const groups = answer?.groups
+  const assignments = answer?.assignments
+  const candidates = descriptor?.candidates
+  const candidateKeys = new Set()
+  const descriptorRoots = new Set()
+  if (!Array.isArray(candidates)) details.push('descriptor.candidates is missing')
+  else
+    candidates.forEach((candidate, index) => {
+      if (
+        !candidate ||
+        typeof candidate.key !== 'string' ||
+        !candidate.key ||
+        typeof candidate.projectName !== 'string' ||
+        !Array.isArray(candidate.roots) ||
+        candidate.roots.some(
+          (row) =>
+            !Array.isArray(row) ||
+            (row.length !== 3 && row.length !== 4) ||
+            typeof row[0] !== 'string' ||
+            (row.length === 4 && (typeof row[3] !== 'string' || !row[3])),
+        )
+      )
+        details.push(`descriptor.candidates[${index}] is invalid`)
+      else {
+        if (candidateKeys.has(candidate.key))
+          details.push(`descriptor.candidates[${index}].key is duplicated`)
+        candidateKeys.add(candidate.key)
+        for (const row of candidate.roots) {
+          if (descriptorRoots.has(row[0]))
+            details.push(`descriptor.candidates[${index}] repeats a root`)
+          descriptorRoots.add(row[0])
+        }
+      }
+    })
+  const rootOrder = Array.isArray(candidates)
+    ? candidates.flatMap((candidate) =>
+        Array.isArray(candidate?.roots)
+          ? candidate.roots.map((row) => (Array.isArray(row) ? row[0] : null))
+          : [],
+      )
+    : null
+  const catalog = descriptor?.projectCatalog
+  if (!Array.isArray(groups)) details.push('groups must be an array')
+  if (!assignments || typeof assignments !== 'object' || Array.isArray(assignments))
+    details.push('assignments must be an object keyed by candidate key')
+  if (!Array.isArray(catalog)) details.push('descriptor.projectCatalog is missing')
+  if (Array.isArray(catalog) && Array.isArray(candidates))
+    for (const candidate of candidates)
+      for (const row of candidate.roots ?? [])
+        if (row.length === 4 && !catalog.some((entry) => entry.projectId === row[3]))
+          details.push(`root ${row[0]} has an unknown targetProjectId`)
+  if (assignments && typeof assignments === 'object' && !Array.isArray(assignments)) {
+    for (const key of candidateKeys)
+      if (!Object.hasOwn(assignments, key))
+        details.push(`assignments is missing candidate key ${key}`)
+    for (const key of Object.keys(assignments))
+      if (!candidateKeys.has(key)) details.push(`assignments contains unknown candidate key ${key}`)
+  }
+  const ids = new Set()
+  const groupIndexes = new Map()
+  const names = new Map()
+  if (Array.isArray(groups))
+    groups.forEach((group, index) => {
+      if (!group || typeof group.id !== 'string' || !group.id.trim()) {
+        details.push(`groups[${index}].id must be a non-empty string`)
+        return
+      }
+      if (ids.has(group.id))
+        details.push(`groups[${index}].id duplicates groups[${groupIndexes.get(group.id)}].id`)
+      if (
+        Object.keys(group).some(
+          (field) => !['id', 'projectName', 'groupKind', 'targetProjectId'].includes(field),
+        )
+      )
+        details.push(`groups[${index}] contains unsupported fields`)
+      if (
+        group.targetProjectId !== undefined &&
+        (typeof group.targetProjectId !== 'string' || !group.targetProjectId)
+      )
+        details.push(`groups[${index}].targetProjectId is invalid`)
+      ids.add(group.id)
+      groupIndexes.set(group.id, index)
+      if (!['project', 'topic', 'unsorted'].includes(group.groupKind))
+        details.push(`groups[${index}].groupKind is invalid`)
+      if (!text(group.projectName, 240)) details.push(`groups[${index}].projectName is invalid`)
+      if (
+        group.targetProjectId &&
+        Array.isArray(catalog) &&
+        !catalog.some((entry) => entry.projectId === group.targetProjectId)
+      )
+        details.push(`groups[${index}].targetProjectId is unknown`)
+      if (group.targetProjectId && group.groupKind !== 'project')
+        details.push(`groups[${index}].targetProjectId requires groupKind project`)
+      if (
+        group.groupKind !== 'unsorted' &&
+        typeof group.projectName === 'string' &&
+        group.projectName.trim()
+      ) {
+        const name = group.projectName.normalize('NFC').toLocaleLowerCase('en-US')
+        if (
+          Array.isArray(catalog) &&
+          catalog.some(
+            (entry) => entry.projectName.normalize('NFC').toLocaleLowerCase('en-US') === name,
+          ) &&
+          !group.targetProjectId
+        )
+          details.push(
+            `groups[${index}].projectName conflicts with a known project; targetProjectId is required`,
+          )
+        const identity = JSON.stringify([group.targetProjectId ?? null, name])
+        const prior = names.get(identity)
+        if (prior)
+          details.push(`groups[${index}].projectName duplicates groups[${prior.index}].projectName`)
+        else names.set(identity, { index, targetProjectId: group.targetProjectId })
+      }
+    })
+  if (assignments && typeof assignments === 'object' && !Array.isArray(assignments))
+    for (const [key, id] of Object.entries(assignments))
+      if (typeof id !== 'string' || !ids.has(id))
+        details.push(`assignments[${key}] names an unknown group id`)
+  const roots = new Set(rootOrder ?? [])
+  const overrides = answer?.overrides ?? []
+  if (!Array.isArray(overrides)) details.push('overrides must be an array when present')
+  const overridden = new Set()
+  if (Array.isArray(overrides))
+    overrides.forEach((override, index) => {
+      if (!override || typeof override.rootKey !== 'string' || !roots.has(override.rootKey))
+        details.push(`overrides[${index}].rootKey is unknown`)
+      else if (overridden.has(override.rootKey))
+        details.push(`overrides[${index}].rootKey is duplicated`)
+      else overridden.add(override.rootKey)
+      if (!override || typeof override.groupId !== 'string' || !ids.has(override.groupId))
+        details.push(`overrides[${index}].groupId names an unknown group id`)
+    })
+  if (
+    Array.isArray(candidates) &&
+    assignments &&
+    typeof assignments === 'object' &&
+    !Array.isArray(assignments) &&
+    Array.isArray(overrides) &&
+    candidates.every((candidate) => Array.isArray(candidate?.roots))
+  ) {
+    const finalIds = new Map()
+    const requiredTargets = new Map()
+    candidates.forEach((candidate) => {
+      for (const row of candidate.roots) {
+        finalIds.set(
+          row[0],
+          Object.hasOwn(assignments, candidate.key) ? assignments[candidate.key] : null,
+        )
+        if (row[3]) requiredTargets.set(row[0], row[3])
+      }
+    })
+    for (const override of overrides)
+      if (typeof override?.rootKey === 'string' && typeof override?.groupId === 'string')
+        finalIds.set(override.rootKey, override.groupId)
+    for (const [rootKey, targetProjectId] of requiredTargets) {
+      const group = Array.isArray(groups)
+        ? groups.find((entry) => entry.id === finalIds.get(rootKey))
+        : null
+      if (group && group.targetProjectId !== targetProjectId)
+        details.push(`root ${rootKey} must remain assigned to targetProjectId ${targetProjectId}`)
+    }
+    const populated = new Set(finalIds.values())
+    for (const [id, index] of groupIndexes)
+      if (!populated.has(id)) details.push(`groups[${index}] has no roots after overrides`)
+  }
+  return { code: details.length ? 'CLASSIFICATION_CONSOLIDATION_INVALID' : null, details }
+}
+
+export function compileAssignments(candidates, assignments, overrides = []) {
+  const result = new Map()
+  for (const candidate of candidates)
+    for (const row of candidate.roots)
+      if (Object.hasOwn(assignments, candidate.key)) result.set(row[0], assignments[candidate.key])
+  for (const override of overrides) result.set(override.rootKey, override.groupId)
+  return result
+}

package/skills/tacitt/scripts/journey-classification.mjs

@@ -0,0 +1,145 @@
+import { readFile } from 'node:fs/promises'
+import { join } from 'node:path'
+import { optional, save } from './journey-state.mjs'
+import { answers, task, taskTokens } from './journey-tasks.mjs'
+import { classifyInstruction } from './journey-classification-tasks.mjs'
+
+export const classificationFormat = 13
+
+/** Requested evidence is bounded to two pages and generates replacement judgments. */
+export async function classificationEvidence(directory, manifest, view, call, limit) {
+  const state = manifest.classification
+  if (!state?.complete) throw new Error('CLASSIFICATION_READS_INCOMPLETE')
+  const created = []
+  let batch = []
+  const evidenceInstruction = `${classifyInstruction} This is the one bounded evidence review for each supplied family; replace each prior judgment. Structural evidence and selected excerpts are partial and do not establish privacy coverage. Return no further evidence requests. If ownership remains unclear, retain the best supported identity/work intent and state the unresolved question in groupingBasis; do not invent project identity. Final reconciliation can retain this family in Unsorted.`
+  const fits = (families) =>
+    taskTokens(directory, `evidence-${families[0].rootKey}`, evidenceInstruction, {
+      knownProjects: manifest.classification.projects.map(({ projectId, projectName }) => ({
+        projectId,
+        projectName,
+      })),
+      families,
+    }) <= 12000
+  const flush = async () => {
+    if (!batch.length) return
+    const evidenceTask = await task(
+      directory,
+      `evidence-${batch[0].rootKey}`,
+      evidenceInstruction,
+      {
+        knownProjects: manifest.classification.projects.map(({ projectId, projectName }) => ({
+          projectId,
+          projectName,
+        })),
+        families: batch,
+      },
+    )
+    const assigned = {
+      ...evidenceTask,
+      rootKeys: batch.map((family) => family.rootKey),
+      evidenceRound: 1,
+    }
+    manifest.tasks.push(assigned)
+    created.push(assigned)
+    state.evidence.push(
+      ...batch.map((family) => ({
+        rootKey: family.rootKey,
+        status: family.projectEvidence.status,
+      })),
+    )
+    await save(join(directory, 'manifest.json'), manifest)
+    batch = []
+  }
+  for (const item of manifest.tasks) {
+    if (item.projectKeys || item.evidenceRound > 0 || !(await optional(item.answer))) continue
+    const reply = await answers([item], 'judgments')
+    const value = JSON.parse(await readFile(item.answer, 'utf8'))
+    const requested = value.evidenceRootKeys ?? []
+    if (
+      !Array.isArray(requested) ||
+      new Set(requested).size !== requested.length ||
+      requested.some((key) => !item.rootKeys.includes(key))
+    )
+      throw new Error('INVALID_EVIDENCE_ROOTS')
+    for (const key of requested) {
+      if (state.evidence.some((entry) => entry.rootKey === key)) continue
+      if (created.length >= limit)
+        return {
+          stage: 'classification',
+          tasks: created,
+          created: created.length,
+          evidenceComplete: false,
+        }
+      const root = state.roots.find((entry) => entry.key === key)
+      const source = state.unknown.find((entry) => entry.rootKey === key)
+      if (!root || !source) throw new Error('INVALID_EVIDENCE_ROOTS')
+      const page = await call('journey.classificationSessions', {
+        journeyId: view.journeyId,
+        workId: root.workId,
+        includeProjectEvidence: true,
+        projectEvidenceRootSessionIds: [root.rootSessionId],
+        rootsOnly: true,
+        limit: 100,
+      })
+      const family = (page.sessions ?? []).filter(
+        (session) => session.rootId === root.rootSessionId,
+      )
+      if (!family.length || family.some((session) => session.rootId !== root.rootSessionId))
+        throw new Error('CLASSIFICATION_EVIDENCE_INVALID')
+      const projectEvidence = family.find(
+        (session) => session.id === root.rootSessionId,
+      )?.projectEvidence
+      if (!projectEvidence) throw new Error('CLASSIFICATION_EVIDENCE_UNAVAILABLE')
+      const rows = []
+      if (projectEvidence.status !== 'supported') {
+        let cursor
+        for (let pageNumber = 0; pageNumber < 2; pageNumber++) {
+          const content = await call('journey.classificationRead', {
+            journeyId: view.journeyId,
+            workId: root.workId,
+            rootSessionId: root.rootSessionId,
+            ...(cursor ? { cursor } : {}),
+          })
+          for (const row of content.rows ?? []) {
+            if (rows.length >= 4) break
+            rows.push({ ...row, text: row.text.slice(0, 800), excerpt: row.text.length > 800 })
+          }
+          cursor = content.cursor
+          if (!cursor || rows.length >= 4) break
+        }
+      }
+      const prior = reply.find((entry) => entry.rootKey === key)
+      const evidenceData = { rootKey: key, metadata: source.metadata, prior, projectEvidence, rows }
+      if (!fits([evidenceData])) throw new Error('CLASSIFICATION_EVIDENCE_TOO_LARGE')
+      if (!fits([...batch, evidenceData])) {
+        await flush()
+        if (created.length >= limit)
+          return {
+            stage: 'classification',
+            tasks: created,
+            created: created.length,
+            evidenceComplete: false,
+          }
+      }
+      batch.push(evidenceData)
+    }
+  }
+  await flush()
+  const pendingEvidence = []
+  for (const item of manifest.tasks) {
+    if (item.evidenceRound > 0 && !(await optional(item.answer))) pendingEvidence.push(item)
+  }
+  let allAnswered = true
+  for (const item of manifest.tasks)
+    if (!item.projectKeys && !(await optional(item.answer))) allAnswered = false
+  return {
+    stage: 'classification',
+    tasks: [
+      ...created,
+      ...pendingEvidence.filter((item) => !created.some((entry) => entry.id === item.id)),
+    ],
+    created: created.length,
+    evidenceComplete: pendingEvidence.length === 0 && allAnswered,
+  }
+}

package/skills/tacitt/scripts/journey-consolidation-submit.mjs

@@ -0,0 +1,190 @@
+import { readFile } from 'node:fs/promises'
+import { createHash } from 'node:crypto'
+import { answers } from './journey-tasks.mjs'
+import {
+  validateClassificationProject,
+  validateConsolidationAnswer,
+  compileAssignments,
+} from './journey-classification-validation.mjs'
+
+/** Expand short handles and fixed catalog membership only after exact coverage validation. */
+export async function classificationSubmission(manifest) {
+  const classification = manifest.classification
+  if (!classification?.complete || !manifest.consolidation)
+    throw new Error('CONSOLIDATION_REQUIRED')
+  const projectMetadata = new Map()
+  for (const item of manifest.tasks.filter((entry) => entry.projectKeys)) {
+    const results = await answers([item], 'items')
+    const ids = results.map((result) => result.projectId)
+    const expected = item.projectKeys.map(
+      (key) => classification.projects.find((project) => project.key === key)?.projectId,
+    )
+    if (
+      results.length !== expected.length ||
+      expected.some((id) => !id) ||
+      new Set(ids).size !== ids.length ||
+      expected.some((id) => !ids.includes(id))
+    )
+      throw new Error('PROJECT_METADATA_COVERAGE_INVALID')
+    for (const result of results) {
+      validateClassificationProject(result, result.projectId)
+      if (projectMetadata.has(result.projectId))
+        throw new Error('PROJECT_METADATA_COVERAGE_INVALID')
+      projectMetadata.set(result.projectId, result)
+    }
+  }
+  const result = classification.projects.map((project) => {
+    const metadata = projectMetadata.get(project.projectId)
+    if (!metadata) throw new Error('PROJECT_METADATA_COVERAGE_INVALID')
+    return {
+      projectName: project.projectName,
+      description: metadata.description,
+      projectType: metadata.projectType,
+      rootSessionIds: project.rootKeys.map((key) => {
+        const root = classification.roots.find((entry) => entry.key === key)
+        if (!root) throw new Error('CLASSIFICATION_ROOT_COVERAGE_INVALID')
+        return root.rootSessionId
+      }),
+      recommendation: metadata.recommendation,
+      projectId: project.projectId,
+    }
+  })
+  let answerText
+  let unknown
+  if (manifest.consolidation.automatic) unknown = { groups: [], assignments: {} }
+  else {
+    answerText = await readFile(manifest.consolidation.answer, 'utf8')
+    unknown = JSON.parse(answerText)
+  }
+  if (!manifest.consolidation.automatic) {
+    const digest = createHash('sha256').update(answerText).digest('hex')
+    if (!manifest.consolidation.answerHash || digest !== manifest.consolidation.answerHash)
+      throw new Error('CLASSIFICATION_CONSOLIDATION_CHANGED')
+  }
+  const candidates = manifest.consolidation.candidates ?? []
+  const rootOrder = candidates.flatMap((candidate) => candidate.roots.map((row) => row[0]))
+  const groups = unknown?.groups
+  const assignments = unknown?.assignments
+  const validation = validateConsolidationAnswer(unknown, manifest.consolidation)
+  const diagnostics = [...validation.details]
+  const expectedRootKeys = classification.unknown
+    .map((item) => item.rootKey)
+    .toSorted((left, right) => left.localeCompare(right))
+  const suppliedRootKeys = [...rootOrder].toSorted((left, right) => left.localeCompare(right))
+  if (
+    rootOrder.length !== expectedRootKeys.length ||
+    new Set(rootOrder).size !== rootOrder.length ||
+    JSON.stringify(suppliedRootKeys) !== JSON.stringify(expectedRootKeys)
+  )
+    diagnostics.push('task root order does not match current classification roots')
+  if (diagnostics.length)
+    throw Object.assign(new Error('CLASSIFICATION_CONSOLIDATION_INVALID'), {
+      code: 'CLASSIFICATION_CONSOLIDATION_INVALID',
+      details: diagnostics,
+    })
+  const groupById = new Map(groups.map((group) => [group.id, group]))
+  const expandedAssignments = compileAssignments(candidates, assignments, unknown.overrides ?? [])
+  const counts = new Map()
+  for (const id of expandedAssignments.values()) counts.set(id, (counts.get(id) ?? 0) + 1)
+  const merged = new Map()
+  const unsorted = []
+  rootOrder.forEach((key) => {
+    const group = groupById.get(expandedAssignments.get(key))
+    const useUnsorted =
+      group.groupKind === 'unsorted' ||
+      (!group.targetProjectId && group.groupKind === 'topic' && counts.get(group.id) < 3)
+    if (useUnsorted) unsorted.push(key)
+    else merged.set(group.id, [...(merged.get(group.id) ?? []), key])
+  })
+  const proposals = [...merged].map(([id, rootKeys]) => ({ ...groupById.get(id), rootKeys }))
+  if (unsorted.length)
+    proposals.push({
+      id: 'unsorted',
+      projectName: 'Unsorted',
+      groupKind: 'unsorted',
+      rootKeys: unsorted,
+    })
+  const rootByKey = new Map(classification.roots.map((root) => [root.key, root]))
+  const metadataMap = new Map()
+  for (const item of manifest.consolidation.metadataTasks ?? []) {
+    if (item.reconciliationHash !== manifest.consolidation.answerHash)
+      throw new Error('CLASSIFICATION_CONSOLIDATION_CHANGED')
+    const descriptor = JSON.parse(await readFile(item.descriptor, 'utf8'))
+    if (descriptor.reconciliationHash !== manifest.consolidation.answerHash)
+      throw new Error('CLASSIFICATION_CONSOLIDATION_CHANGED')
+    const values = await answers([item], 'items')
+    if (
+      !Array.isArray(item.projectIds) ||
+      values.length !== item.projectIds.length ||
+      new Set(item.projectIds).size !== item.projectIds.length
+    )
+      throw new Error('PROJECT_METADATA_COVERAGE_INVALID')
+    for (const value of values) {
+      validateClassificationProject(value, value.projectId)
+      if (!item.projectIds.includes(value.projectId) || metadataMap.has(value.projectId))
+        throw new Error('PROJECT_METADATA_COVERAGE_INVALID')
+      metadataMap.set(value.projectId, value)
+    }
+  }
+  const requiredMetadataIds = new Set(
+    proposals.filter((item) => !item.targetProjectId).map((item) => item.id),
+  )
+  if (
+    requiredMetadataIds.size !== metadataMap.size ||
+    [...requiredMetadataIds].some((id) => !metadataMap.has(id))
+  )
+    throw new Error('PROJECT_METADATA_COVERAGE_INVALID')
+  for (const item of proposals) {
+    const rootSessionIds = item.rootKeys.map((key) => rootByKey.get(key)?.rootSessionId)
+    if (rootSessionIds.some((id) => !id)) throw new Error('CLASSIFICATION_ROOT_COVERAGE_INVALID')
+    const {
+      id: _id,
+      rootKeys: _rootKeys,
+      groupKind: _groupKind,
+      targetProjectId,
+      ...metadata
+    } = item
+    if (targetProjectId) {
+      const target = result.find((entry) => entry.projectId === targetProjectId)
+      if (!target) throw new Error('UNKNOWN_TARGET_PROJECT')
+      target.rootSessionIds.push(...rootSessionIds)
+    } else {
+      const submittedMetadata = metadataMap.get(item.id)
+      if (!submittedMetadata) throw new Error('PROJECT_METADATA_COVERAGE_INVALID')
+      result.push({
+        projectName: metadata.projectName,
+        description: submittedMetadata.description,
+        projectType: submittedMetadata.projectType,
+        recommendation: submittedMetadata.recommendation,
+        rootSessionIds,
+      })
+    }
+  }
+  const expectedRootIds = classification.roots
+    .map((root) => root.rootSessionId)
+    .toSorted((left, right) => left.localeCompare(right))
+  const submittedRootIds = result
+    .flatMap((item) => item.rootSessionIds)
+    .toSorted((left, right) => left.localeCompare(right))
+  if (
+    new Set(submittedRootIds).size !== classification.roots.length ||
+    JSON.stringify(expectedRootIds) !== JSON.stringify(submittedRootIds)
+  )
+    throw new Error('CLASSIFICATION_ROOT_COVERAGE_INVALID')
+  const normalizedNames = new Map()
+  const output = result.map(({ projectId: _projectId, ...item }) => ({
+    ...item,
+    projectName: item.projectName.trim(),
+    description: item.description.trim(),
+    recommendation: { ...item.recommendation, reason: item.recommendation.reason.trim() },
+  }))
+  for (const item of output) {
+    const key = item.projectName.normalize('NFC').toLocaleLowerCase('en-US')
+    const metadata = JSON.stringify([item.description, item.projectType, item.recommendation])
+    const prior = normalizedNames.get(key)
+    if (prior !== undefined && prior !== metadata)
+      throw new Error('CLASSIFICATION_PROJECT_NAME_CONFLICT')
+    normalizedNames.set(key, metadata)
+  }
+  return output
+}

package/skills/tacitt/scripts/journey-consolidation.d.mts

@@ -0,0 +1,9 @@
+export function consolidateClassification(
+  directory: string,
+  manifest: Record<string, unknown>,
+  options?: { refresh?: boolean },
+): Promise<Record<string, unknown>>
+
+export function classificationSubmission(
+  manifest: Record<string, unknown>,
+): Promise<Array<Record<string, unknown>>>