dsh-model-everything 0.0.0-stage → 0.2.0

package/src/baseline.ts

@@ -0,0 +1,288 @@
+/**
+ * Report diffing: "did this slice add findings, or remove them?"
+ *
+ * The acceptance criterion of a refactoring slice is usually *not* "zero findings" —
+ * it is "no NEW findings, and none of the old ones got worse". Comparing two JSON
+ * reports by hand is how that gets fudged, so the comparison is code.
+ *
+ * A finding's identity is its machine-readable locator, never its prose:
+ * `check|code|<canonical evidence + path>`. Findings that carry neither evidence nor a
+ * path fall back to their message, because an engine that reports nothing structured
+ * has nothing stabler to match on. Two runs over the same input must produce zero
+ * added and zero removed findings.
+ *
+ * @module dsh-model-everything-baseline
+ */
+
+import { canonicalJson, REPORT_SCHEMAS, verdictOf, type Finding, type Verdict, type VerdictSummary } from './engine.js'
+
+/** A finding as it appears in any report family, after flattening. */
+export interface ReportFinding extends Finding {
+  /** The check that emitted it (verify/datamodel: `S2`, `C1`, …); absent for flat families. */
+  check?: string
+}
+
+export interface BaselineChange {
+  identity: string
+  code: string
+  before: { severity: string; message: string }
+  after: { severity: string; message: string }
+}
+
+export interface BaselineReport {
+  ok: boolean
+  ran: boolean
+  verdict: Verdict
+  verdictReason: string
+  /** The versioned report contract this result follows. */
+  schema: string
+  /** The current report's own verdict, echoed so "no new findings" cannot hide a failing run. */
+  currentVerdict: Verdict
+  added: ReportFinding[]
+  removed: ReportFinding[]
+  changed: BaselineChange[]
+  summary: {
+    baseline: number
+    current: number
+    added: number
+    removed: number
+    changed: number
+    addedErrors: number
+    addedWarnings: number
+  }
+  nextSteps: string[]
+}
+
+/**
+ * The stable identity of a finding: check + file + code + a canonical locator. Prose is
+ * excluded so a reworded message reads as `changed`, not as a new finding.
+ */
+export function findingIdentity(finding: ReportFinding): string {
+  const locator = canonicalJson({
+    ...(finding.evidence === undefined ? {} : { evidence: finding.evidence }),
+    ...(finding.path === undefined ? {} : { path: finding.path }),
+  })
+  const where = locator === '{}' ? 'message:' + finding.message : locator
+  const file = finding.file === undefined ? '' : finding.file + '|'
+  return (finding.check === undefined ? '' : finding.check + '|') + file + finding.code + '|' + where
+}
+
+/** Flatten a report of any family into one finding list (`checks[].findings` or `findings[]`). */
+export function flattenReportFindings(report: unknown): ReportFinding[] {
+  if (report === null || typeof report !== 'object') return []
+  const record = report as Record<string, unknown>
+  const flat: ReportFinding[] = []
+  if (Array.isArray(record.checks)) {
+    for (const check of record.checks as Array<Record<string, unknown>>) {
+      const id = typeof check.id === 'string' ? check.id : undefined
+      for (const finding of (Array.isArray(check.findings) ? check.findings : []) as Finding[]) {
+        flat.push(id === undefined ? finding : { ...finding, check: id })
+      }
+    }
+  }
+  if (Array.isArray(record.findings)) {
+    for (const finding of record.findings as Finding[]) flat.push(finding)
+  }
+  return flat
+}
+
+/** The report's own verdict, whatever family it came from. */
+function verdictOfReport(report: unknown): Verdict {
+  if (report === null || typeof report !== 'object') return 'fail'
+  const value = (report as Record<string, unknown>).verdict
+  return value === 'pass' || value === 'pass_with_findings' || value === 'fail' ? value : 'fail'
+}
+
+/**
+ * The delta's verdict, from the four ways a comparison can get worse or better.
+ *
+ * `escalated` is the one that used to be missing: a finding that keeps its identity and moves from warning to
+ * error lives in `changed`, not in `added`, so folding severities into `addedErrors` only left the comparison
+ * answering `pass` over a change that made the report strictly worse. A de-escalation is not a regression: it is
+ * still a change, so it is named, and the verdict stays `pass`.
+ */
+function diffOf(added: number, removed: number, addedErrors: number, escalated: number, deescalated: number, firstCode?: string): VerdictSummary {
+  const errors = addedErrors + escalated
+  if (errors > 0) {
+    const bits: string[] = []
+    if (added > 0) bits.push(String(added) + ' new finding(s)')
+    if (escalated > 0) bits.push(String(escalated) + ' finding(s) escalated to error')
+    return {
+      verdict: 'fail',
+      // The count of errors is the sum: a reader must not have to add "new" and "escalated" in their head.
+      verdictReason: bits.join(', ') + ' (' + String(errors) + ' error)' + (firstCode === undefined ? '' : ' (first error: ' + firstCode + ')'),
+    }
+  }
+  if (added > 0) {
+    return {
+      verdict: 'pass_with_findings',
+      verdictReason: String(added) + ' new warning finding(s), no new errors' + (deescalated > 0 ? '; ' + String(deescalated) + ' finding(s) moved to a lower severity' : ''),
+    }
+  }
+  if (removed > 0) {
+    return { verdict: 'pass', verdictReason: 'no new findings; ' + String(removed) + ' finding(s) resolved' }
+  }
+  if (deescalated > 0) {
+    return { verdict: 'pass', verdictReason: 'no new findings; ' + String(deescalated) + ' finding(s) moved to a lower severity' }
+  }
+  return { verdict: 'pass', verdictReason: 'no new findings, nothing resolved' }
+}
+
+/**
+ * Compare a baseline report with the current one. The verdict describes the *delta*
+ * (a new error fails it), while `currentVerdict` keeps the absolute result visible.
+ */
+/**
+ * Whether a value is a report this can compare.
+ *
+ * `diffReports(null, null)` answered `verdict: 'pass'`. With no inputs, every comparison found
+ * nothing to compare and the report said the two agree - the worst shape of a false green, because
+ * the absence of an input reads as agreement. A caller who passed the wrong variable would take
+ * that verdict at face value.
+ *
+ * The first fix refused only `null` and `undefined`, on the argument that a present-but-wrong value
+ * still carries `currentVerdict: 'fail'` and the delta is honestly empty. Running the installed
+ * plugin disproved that: `report_diff` with two `'junk'` strings answered `verdict: pass`, and
+ * `verdict` is the field a reader is told to act on. A mitigation in a field the reader is told to
+ * ignore is not a mitigation.
+ *
+ * The second fix required "an object", which moved the hole rather than closing it: `{}` is an object,
+ * has no findings, and two of them compare as agreement. A guard is only as deep as its requirement,
+ * so the requirement is now what the name promises - an object that carries at least one report field.
+ * A partial report still passes, because `findings` alone is one.
+ */
+const REPORT_FIELDS = ['schema', 'verdict', 'ran', 'ok', 'checks', 'findings', 'summary', 'hashes', 'currentVerdict', 'added', 'removed']
+
+function requireReport(value: unknown, role: string): Record<string, unknown> {
+  if (value === null || value === undefined) {
+    throw new Error(role + ' is ' + String(value) + '. A diff needs both sides: with no input at all, every comparison finds nothing and the report reads as agreement between two things that were never compared.')
+  }
+  if (typeof value !== 'object' || Array.isArray(value)) {
+    throw new Error(role + ' is a ' + typeof value + (Array.isArray(value) ? ' (an array)' : '') + ', not a report. A diff needs two report objects: two values that are not reports flatten to two empty finding lists, and the delta then reads as agreement between things that were never compared.')
+  }
+  const record = value as Record<string, unknown>
+  if (!REPORT_FIELDS.some((field) => field in record)) {
+    const keys = Object.keys(record)
+    throw new Error(role + ' is an object with no report fields' + (keys.length === 0 ? ' at all' : ' (' + keys.slice(0, 6).join(', ') + ')') + ', not a report. Two such values flatten to two empty finding lists, and the delta then reads as agreement between things that were never compared.')
+  }
+  return record
+}
+
+/**
+ * Why a report-shaped value still must not be compared, or `undefined` if it can be.
+ *
+ * Two shapes reach here legitimately and neither is a report about a model:
+ *
+ * - a REFUSAL (`ran: false`): the tool did not run, so there is nothing to compare against;
+ * - a report about an INVALID INPUT (`summary.checksRun === 0`): the engine ran, refused the input, and
+ *   examined nothing. Its finding list is not empty because the model is clean, and comparing it against
+ *   a real report would call every finding in that report resolved.
+ *
+ * `checksRun` is read only when the report carries it, because the families that report flat findings
+ * have no check roster and `compose` counts machines rather than checks.
+ */
+function incomparableReason(record: Record<string, unknown>, role: string): string | undefined {
+  if (record.ran === false) {
+    return role + ' is a refusal: the fields describe why the tool did not run, so there is no report to compare'
+  }
+  const summary = record.summary
+  if (summary !== null && typeof summary === 'object' && (summary as Record<string, unknown>).checksRun === 0) {
+    return role + ' examined nothing (its summary says checksRun 0), so its empty finding list is not evidence of anything'
+  }
+  return undefined
+}
+
+/** The family a report belongs to, when it says so. Two different schemas cannot be compared. */
+function familyOf(record: Record<string, unknown>): string | undefined {
+  return typeof record.schema === 'string' && record.schema !== '' ? record.schema : undefined
+}
+
+export function diffReports(baseline: unknown, current: unknown): BaselineReport {
+  const beforeRecord = requireReport(baseline, 'baseline')
+  const afterRecord = requireReport(current, 'current')
+  for (const [record, role] of [[beforeRecord, 'baseline'], [afterRecord, 'current']] as const) {
+    const reason = incomparableReason(record, role)
+    if (reason !== undefined) throw new Error(reason + '. A diff needs two runs that produced reports.')
+  }
+  const beforeFamily = familyOf(beforeRecord)
+  const afterFamily = familyOf(afterRecord)
+  if (beforeFamily !== undefined && afterFamily !== undefined && beforeFamily !== afterFamily) {
+    throw new Error('cannot compare a ' + beforeFamily + ' report with a ' + afterFamily + ' report: they do not share a finding vocabulary, so every finding on one side would read as resolved or added on the other. Compare two runs of the same tool.')
+  }
+  const before = flattenReportFindings(baseline)
+  const after = flattenReportFindings(current)
+  const beforeByIdentity = new Map<string, ReportFinding>()
+  for (const finding of before) beforeByIdentity.set(findingIdentity(finding), finding)
+  const seen = new Set<string>()
+  const added: ReportFinding[] = []
+  const changed: BaselineChange[] = []
+  for (const finding of after) {
+    const identity = findingIdentity(finding)
+    seen.add(identity)
+    const previous = beforeByIdentity.get(identity)
+    if (previous === undefined) {
+      added.push(finding)
+      continue
+    }
+    if (previous.severity !== finding.severity || previous.message !== finding.message) {
+      changed.push({
+        identity,
+        code: finding.code,
+        before: { severity: previous.severity, message: previous.message },
+        after: { severity: finding.severity, message: finding.message },
+      })
+    }
+  }
+  const removed = before.filter((finding) => !seen.has(findingIdentity(finding)))
+  const addedErrors = added.filter((finding) => finding.severity === 'error')
+  const addedWarnings = added.filter((finding) => finding.severity === 'warning')
+  // Severity movement lives in `changed`. Counted here, because the verdict is about whether the slice got worse,
+  // and a warning that became an error did - even though nothing was added.
+  const escalated = changed.filter((change) => change.before.severity !== 'error' && change.after.severity === 'error')
+  const deescalated = changed.filter((change) => change.before.severity === 'error' && change.after.severity !== 'error')
+  const delta = diffOf(added.length, removed.length, addedErrors.length, escalated.length, deescalated.length, addedErrors[0]?.code ?? escalated[0]?.code)
+
+  const nextSteps: string[] = []
+  if (addedErrors.length > 0) {
+    nextSteps.push('Fix or explicitly accept the ' + String(addedErrors.length) + ' new error finding(s) before calling this slice done: the baseline is what "not worse" is measured against.')
+  } else if (escalated.length > 0) {
+    nextSteps.push('A finding moved from warning to error (' + String(escalated.length) + '): the slice added nothing but still made the report worse, so the baseline is less strict than the run.')
+  } else if (added.length > 0) {
+    nextSteps.push('The new warnings are not failures, but they are new: decide per finding whether the change introduced them or the baseline was incomplete.')
+  } else {
+    nextSteps.push('No new findings: re-record the baseline with this report if the slice is accepted, so the next comparison starts from it.')
+  }
+  if (changed.length > 0) nextSteps.push('Review the ' + String(changed.length) + ' changed finding(s): the same locator now reports something different, which usually means the fix moved the problem.')
+  if (removed.length > 0) nextSteps.push(String(removed.length) + ' finding(s) from the baseline are gone — confirm they were fixed rather than renamed out of the report (a renamed check or a reworded message changes the identity).')
+  if (verdictOfReport(current) === 'fail') {
+    // The sentence has to match the delta it sits beside: "the delta being clean" is false when the delta failed,
+    // which is exactly the case an escalation now produces.
+    const deltaClean = delta.verdict === 'pass' || delta.verdict === 'pass_with_findings'
+    nextSteps.push(deltaClean
+      ? 'The current report still fails on its own terms; the delta being clean does not make the run clean.'
+      : 'The current report still fails on its own terms, and the delta already says the slice made it worse: settle both before accepting it.')
+  }
+
+  return {
+    ok: true,
+    ran: true,
+    ...delta,
+    schema: REPORT_SCHEMAS.baseline,
+    currentVerdict: verdictOfReport(current),
+    added,
+    removed,
+    changed,
+    summary: {
+      baseline: before.length,
+      current: after.length,
+      added: added.length,
+      removed: removed.length,
+      changed: changed.length,
+      addedErrors: addedErrors.length,
+      addedWarnings: addedWarnings.length,
+    },
+    nextSteps,
+  }
+}
+
+export type { VerdictSummary }

package/src/compose-tool.ts

@@ -0,0 +1,82 @@
+import { defineTool } from '@deepseek-ai/dsh-tools'
+import type { JsonValue } from './json-value.js'
+import { runCompositionVerification } from './engine.js'
+import type { SharedAccess, SharedLocation } from './engine.js'
+import { OptionShapeError, optionRefusal, requireArrayOption } from './option-shape.js'
+import { renderReportText } from './report-view.js'
+
+export const COMPOSE_TOOL_NAME = 'model_compose_verify'
+
+/**
+ * DSH tool wrapping multi-machine composition verification:
+ * product-space reachability with rendezvous handshakes (C1 deadlock, C2 sync), and the shared
+ * locations the machines both touch (C3 declarations, C4 unordered write, C5 non-atomic update).
+ */
+export const modelComposeTool = defineTool({
+  name: COMPOSE_TOOL_NAME,
+  description:
+    'Check two or more LogicModelV1 machines together. Pass the models in `machines`. Pass `rendezvous` for handshake events: these fire only when every declaring machine is enabled at once, and a terminal machine stops taking part. Other events advance one machine, so a machine waiting on a handshake can take any number of its own steps first. There is no rate ratio and no fairness bound. Pass `shared` for locations two machines both touch and `accesses` to say which machine event reads or writes one: C3 reports a declaration that names something the composition does not have, C4 an unordered write to one location both machines write, and C5 a read-modify-write pair the model does not make atomic (the lost-update interleaving). Two accesses that name the same `protectedBy` lock are treated as exclusive; that the lock is held across the whole operation is a single-machine question this tool does not answer. Returns C1 deadlocks (no machine can advance, and one is not terminal) and C2 handshakes that never fire. Both name the reason per machine, and C1 gives the shortest counterexample. `maxStates` caps the search; a capped run says so. It decides what no single machine can show: a handshake that never fires, or a node where no machine may advance because of the others. One machine on its own is `model_verify`. Read `verdict` and `verdictReason`.',
+  parameters: {
+    machines: {
+      type: 'json',
+      required: true,
+      description: 'Array of LogicModelV1 state-machine models to compose (two or more).',
+    },
+    rendezvous: {
+      type: 'json',
+      description: 'Optional array of handshake event names shared by the machines.',
+    },
+    shared: {
+      type: 'json',
+      description: 'Optional array of shared locations: { id, initial? }. Without one, nothing is shared between the machines.',
+    },
+    accesses: {
+      type: 'json',
+      description: 'Optional array of { machine, event, location, kind: "read" | "write" | "read-write", protectedBy? }. A read-write is a read-modify-write: two of them that are not mutually exclusive lose an update.',
+    },
+    maxStates: {
+      type: 'integer',
+      description: 'Maximum composite states to explore. Default 10000.',
+    },
+    check: {
+      type: 'string',
+      description: 'Print one check in full when the digest truncated it. Pass the check id the CHECKS line lists (for example A9); the per-check budget no longer applies, and an id this report does not carry is answered with the ids that it does.',
+    },
+    reportView: {
+      type: 'string',
+      enum: ['digest', 'full'],
+      description: 'digest (default) prints the readable body; full prints the structured report, which report_diff needs.',
+    },
+  },
+  output: {
+    schema: {
+      type: 'json',
+      description: 'Composition verification report with C1/C2 findings and, when shared locations are declared, C3/C4/C5.',
+    },
+    render(args, value) {
+      return [{ type: 'text' as const, text: renderReportText(value, { full: args.reportView === 'full', check: args.check }) }]
+    },
+  },
+  timeoutMs: 10_000,
+  isConcurrencySafe: () => true,
+  async execute(args) {
+    try {
+      // `machines` is declared `json`, so a string reached `machinesInput.forEach` as a raw TypeError. The
+      // array-ness is the tool's contract; check it here and refuse with a code that names the option.
+      requireArrayOption(args.machines, 'machines', 'COMPOSE_OPTION_INVALID')
+      if (args.machines !== undefined && (args.machines as unknown[]).length === 0) {
+        throw new OptionShapeError('COMPOSE_OPTION_INVALID', 'machines is an empty array. A composition needs at least two machines to compose; with none there is nothing to decide.')
+      }
+      if (args.shared !== undefined) requireArrayOption(args.shared, 'shared', 'COMPOSE_OPTION_INVALID')
+      if (args.accesses !== undefined) requireArrayOption(args.accesses, 'accesses', 'COMPOSE_OPTION_INVALID')
+      return runCompositionVerification(args.machines as unknown[], {
+        rendezvous: args.rendezvous as string[] | undefined,
+        shared: args.shared as SharedLocation[] | undefined,
+        accesses: args.accesses as SharedAccess[] | undefined,
+        maxStates: args.maxStates,
+      }) as unknown as JsonValue
+    } catch (error) {
+      return optionRefusal(error, 'COMPOSE_OPTION_INVALID') as unknown as JsonValue
+    }
+  },
+})

package/src/concurrency-tool.ts

@@ -0,0 +1,44 @@
+import { defineTool } from '@deepseek-ai/dsh-tools'
+import type { JsonValue } from './json-value.js'
+import { runConcurrencyScan } from './concurrency.js'
+import { renderReportText } from './report-view.js'
+
+export const INTERLEAVING_SCAN_TOOL_NAME = 'model_interleaving_scan'
+
+/**
+ * Model-visible DSH tool that mines design documents/plans for concurrency-related
+ * claims and risk keywords. It does not prove concurrency safety; it flags terms
+ * such as "thread-safe", "lock-free", "race condition", "atomic", "mutex", etc.,
+ * so the model can either provide dedicated evidence or mark the claim unverified.
+ */
+export const modelConcurrencyScanTool = defineTool({
+  name: INTERLEAVING_SCAN_TOOL_NAME,
+  description:
+    'Scan a document or plan text for claims about how work is executed: threads, synchronization, async tasks, interrupts, shared state. Use ONLY after confirming the target actually has concurrency requirements or behavior; if the target is purely sequential, do not call this tool. Findings carry a family (threads, synchronization, async, interrupts, shared-state) and an absolute claim is one a plan makes about safety rather than a risk it mentions. Returns keywords like thread-safe, lock-free, wait-free, data race, race condition, atomic, synchronized, mutex, semaphore, spinlock, shared variable, shared memory, reentrant, interrupt-safe, ISR, IRQ, NMI, critical section, disable_irq/enable_irq; async ones such as async, await, promise, event loop, callback, microtask, coroutine, fire-and-forget, backpressure, task queue; and their Chinese counterparts (线程安全, 无数据竞争, 无锁, 互斥, 原子, 并发, 临界区, 异步, 回调, 事件循环, 微任务, 协程, 背压 and so on). Absolute claims (thread-safe, lock-free, no data race, interrupt-safe, ISR-safe, non-blocking, never blocks) are flagged as errors requiring dedicated verification, and each carries suggestions naming the analysis to run instead, chosen by family: TSan/Helgrind/CBMC/TLA+ for threads, RTOS-aware analysis for interrupts, and TLA+/UPPAAL plus a runtime check for async, where a blocking call on the event loop, a missing await and a lock held across an await are invisible to any model. Text with no claim to mine is refused with `CONCURRENCY_NO_CLAIM` rather than reported as clean: a scan of nothing is not a clean bill of health, and that includes text in a vocabulary this scan does not match. This plugin does not prove concurrency safety, and this tool does not check a machine: to decide a protocol it mined, model it and call `model_verify`, or compose two of them with `model_compose_verify`.',
+  parameters: {
+    text: {
+      type: 'string',
+      required: true,
+      description: 'Document or plan text to scan for claims about threads, synchronization, async tasks, interrupts or shared state.',
+    },
+    reportView: {
+      type: 'string',
+      enum: ['digest', 'full'],
+      description: 'digest (default) prints the readable body; full prints the structured report, which report_diff needs.',
+    },
+  },
+  output: {
+    schema: {
+      type: 'json',
+      description: 'Concurrency scan report with findings and summary.',
+    },
+    render(args, value) {
+      return [{ type: 'text' as const, text: renderReportText(value, { full: args.reportView === 'full' }) }]
+    },
+  },
+  timeoutMs: 10_000,
+  isConcurrencySafe: () => true,
+  async execute(args) {
+    return runConcurrencyScan(args.text) as unknown as JsonValue
+  },
+})