@ttsc/evidence 0.30.3 → 0.31.0

package/native/markdown_capture.go

@@ -0,0 +1,223 @@
+package evidence
+
+import (
+  "io/fs"
+  "sort"
+  "strconv"
+  "strings"
+)
+
+// markdownCapture owns the first Markdown observations of one Check. Its
+// inventories retain the first consumed bytes' interpretation and address;
+// walking another population never rereads an already captured address.
+// Different file selections may require another walk, while equal selections
+// replay the original walk, including failed observations and pruning.
+//
+// The host keeps the witness of each actual operation. This capture neither
+// certifies freshness nor restores withdrawn authority. An edit after an
+// observation belongs to a later Check; downstream reusable delivery still
+// validates the original witnesses. Hints projects the same captured snapshot.
+//
+// Base and artifact-address keys keep citation spaces distinct, while ordered
+// file-glob semantics determine traversal sharing. Loaders apply symbol and
+// severity policy anew. Native paths and directory entries come from the
+// ordinary input reader without an inferred case or symlink policy.
+//
+// A capture parses each addressed file once. Replaying recorded entries costs
+// a traversal of those entries without raw reads or parsing; new file selections
+// may require another discovery. Key construction depends on glob text and
+// population sizes, including sorting distinct ordered glob sets.
+//
+// Check releases its maps and traversal records when evaluation ends. Their
+// storage grows with addressed files, parsed bytes, selected traversal entries
+// and distinct file selections. A clean Corpus retains the inventories needed
+// for Hints. The capture owns no descriptor or task.
+type markdownCapture struct {
+  bases        map[string]markdownCapturedBase
+  walks        map[string]markdownCapturedWalk
+  inventories  map[string]map[string]markdownCapturedInventory
+  walkFailures map[string]error
+  walkErrors   map[string]map[string]error
+}
+
+type markdownCapturedBase struct {
+  from    string
+  problem string
+}
+
+type markdownCapturedWalk struct {
+  entries []markdownWalkObservation
+  err     error
+}
+
+type markdownWalkObservation struct {
+  path  string
+  entry fs.DirEntry
+  err   error
+}
+
+type markdownCapturedInventory struct {
+  inventory *artifactInventory
+  err       error
+}
+
+// resolve retains the first base resolution, including an unusable root.
+// Base.Absolute is the existing declared address-space identity, so physical
+// aliases are not collapsed by this capture.
+func (capture *markdownCapture) resolve(base populationBase) (string, string) {
+  if prior, ok := capture.bases[base.Absolute]; ok {
+    return prior.from, prior.problem
+  }
+  observation := markdownCapturedBase{problem: baseDirectoryProblem(base, artifactMarkdown)}
+  if observation.problem == "" {
+    var resolved bool
+    observation.from, resolved = resolvedBaseDirectory(base)
+    if !resolved {
+      observation.problem = unresolvedBaseProblem(base, artifactMarkdown)
+    }
+  }
+  if capture.bases == nil {
+    capture.bases = map[string]markdownCapturedBase{}
+  }
+  capture.bases[base.Absolute] = observation
+  return observation.from, observation.problem
+}
+
+// walk replays only a traversal with the same file-selection semantics. The
+// loader's traversal decisions depend on those globs, not its symbol/severity
+// projection. A new selection delegates discovery to the ordinary reader.
+func (capture *markdownCapture) walk(base populationBase, from string, config graphConfig, visit fs.WalkDirFunc) error {
+  key := markdownWalkKey(base, config)
+  if prior, ok := capture.walks[key]; ok {
+    for _, observation := range prior.entries {
+      if err := visit(observation.path, observation.entry, observation.err); err != nil && err != fs.SkipDir {
+        return err
+      }
+    }
+    return prior.err
+  }
+  if err := capture.walkFailures[base.Absolute]; err != nil {
+    return err
+  }
+  observation := markdownCapturedWalk{}
+  observedErrors := map[string]bool{}
+  observation.err = base.inputs.WalkDir(from, func(path string, entry fs.DirEntry, err error) error {
+    if prior := capture.walkErrors[base.Absolute][path]; prior != nil {
+      err = prior
+    } else if err != nil {
+      if capture.walkErrors == nil {
+        capture.walkErrors = map[string]map[string]error{}
+      }
+      if capture.walkErrors[base.Absolute] == nil {
+        capture.walkErrors[base.Absolute] = map[string]error{}
+      }
+      capture.walkErrors[base.Absolute][path] = err
+    }
+    if err != nil {
+      observedErrors[path] = true
+    }
+    observation.entries = append(observation.entries, markdownWalkObservation{path: path, entry: entry, err: err})
+    return visit(path, entry, err)
+  })
+  // Broader discovery can omit an earlier failed entry after a deletion or
+  // pruning change. Its first failure still belongs to this snapshot; replay
+  // it through the new phase's relevance, severity and population-health gate.
+  if observation.err == nil {
+    for path, err := range capture.walkErrors[base.Absolute] {
+      if observedErrors[path] {
+        continue
+      }
+      observation.entries = append(observation.entries, markdownWalkObservation{path: path, err: err})
+      if result := visit(path, nil, err); result != nil && result != fs.SkipDir {
+        observation.err = result
+        break
+      }
+    }
+  }
+  if observation.err != nil {
+    if capture.walkFailures == nil {
+      capture.walkFailures = map[string]error{}
+    }
+    capture.walkFailures[base.Absolute] = observation.err
+  }
+  if capture.walks == nil {
+    capture.walks = map[string]markdownCapturedWalk{}
+  }
+  capture.walks[key] = observation
+  return observation.err
+}
+
+// read retains successful parses and failed reads alike. The first actual
+// reader invocation supplies the only consumed-byte witness for this address;
+// the capture never retries a failure in another phase of the same Check.
+func (capture *markdownCapture) read(address artifactAddress, current string) (*artifactInventory, error) {
+  if prior, ok := capture.inventories[address.Base.Absolute][address.Relative]; ok {
+    return prior.inventory, prior.err
+  }
+  content, err := address.Base.inputs.ReadFile(current)
+  var inventory *artifactInventory
+  if err == nil {
+    inventory, _ = scanMarkdownInventory(address, string(content))
+  } else {
+    inventory = &artifactInventory{Path: address.Display, Type: artifactMarkdown, LoadFailed: true}
+  }
+  if capture.inventories == nil {
+    capture.inventories = map[string]map[string]markdownCapturedInventory{}
+  }
+  if capture.inventories[address.Base.Absolute] == nil {
+    capture.inventories[address.Base.Absolute] = map[string]markdownCapturedInventory{}
+  }
+  capture.inventories[address.Base.Absolute][address.Relative] = markdownCapturedInventory{inventory: inventory, err: err}
+  return inventory, err
+}
+
+// markdownWalkKey canonicalizes the union of ordered glob sets for one base.
+// Each set keeps include/exclude order; sorting and deduplicating whole sets
+// reflects that different populations are combined by union during traversal.
+func markdownWalkKey(base populationBase, config graphConfig) string {
+  sets := map[string]bool{}
+  add := func(candidate populationBase, files globSet) {
+    if candidate.Absolute != base.Absolute {
+      return
+    }
+    patterns := make([]string, 0, len(files.Patterns))
+    for _, pattern := range files.Patterns {
+      value := strings.Join(pattern.Segments, "/")
+      if pattern.Exclude {
+        value = "1:" + value
+      } else {
+        value = "0:" + value
+      }
+      patterns = append(patterns, value)
+    }
+    var key strings.Builder
+    for _, pattern := range patterns {
+      key.WriteString(strconv.Itoa(len(pattern)))
+      key.WriteByte(':')
+      key.WriteString(pattern)
+    }
+    sets[key.String()] = true
+  }
+  for _, claim := range config.Claims {
+    if claim.Type == artifactMarkdown {
+      add(claim.Base, claim.Files)
+    }
+    for _, reference := range claim.References {
+      if reference.Type == artifactMarkdown {
+        add(reference.Base, reference.Files)
+      }
+    }
+  }
+  keys := make([]string, 0, len(sets))
+  for key := range sets {
+    keys = append(keys, key)
+  }
+  sort.Strings(keys)
+  var key strings.Builder
+  for _, part := range append([]string{base.Absolute}, keys...) {
+    key.WriteString(strconv.Itoa(len(part)))
+    key.WriteByte(':')
+    key.WriteString(part)
+  }
+  return key.String()
+}

package/native/model.go

@@ -54,6 +54,7 @@ const (
 )
 
 type graphConfig struct {
+  inputs evidenceInputReader
   Claims []claimSpec
 }
 
@@ -282,7 +283,7 @@ type evidenceUnit struct {
   // So a nested change moves both its own unit's digest and every enclosing
   // unit's. `scopeIndex` composes ancestors and descendants anyway, so detection
   // is unaffected, but nothing here may assume a unit's digest is independent of
-  // its subtree. A feature built on that assumption was reverted once already.
+  // its subtree.
   //
   // Removing the documentation comment is what stops the review from
   // invalidating itself: writing an `@evidenceReview` inside a unit that is

package/native/prisma.go

@@ -163,15 +163,32 @@ func loadPrismaInventories(
   // resident host repeats this every cycle, so an unchanged schema would
   // otherwise be re-parsed on every TypeScript keystroke that rebuilds.
   severity := prismaSetSeverity(config, inventories)
-  digest := prismaContentDigest(root, set.Sources)
+  // The bridge owns additional module/file reads outside this native reader.
+  // Neither a memo hit nor a later disk hash proves those producer inputs.
+  config.inputs.Unavailable()
+  digest := prismaContentDigest(root, set.Sources, config.inputs)
+  trace := newEvidenceBridgeTrace("prisma")
+  if trace != nil {
+    trace.nativeLookup = true
+    trace.record("bridge-lookup", os.Getpid(), map[string]any{
+      "bridge": "prisma", "root": root, "requestId": prismaSetID,
+      "sources": set.Sources, "nativeDigest": digest, "nativeLookup": true,
+    })
+  }
   if outcome, hit := prismaSchemas.lookup(digest); hit {
+    if trace != nil {
+      trace.record("bridge-cache-hit", os.Getpid(), map[string]any{"bridge": "prisma", "nativeDigest": digest})
+    }
     return inventories, append(
       problems,
       prismaUnitsFromOutcome(root, set, inventories, outcome, config)...,
     )
   }
 
-  result, err := normalizePrismaSet(root, set.Sources)
+  if trace != nil {
+    trace.record("bridge-cache-miss", os.Getpid(), map[string]any{"bridge": "prisma", "nativeDigest": digest})
+  }
+  result, err := normalizePrismaSet(root, set.Sources, trace)
   if err != nil {
     message := "Evidence graph could not run its Prisma schema loader: " + causeText(err) + ". Prisma references require Node.js and a resolvable @prisma/prisma-schema-wasm."
     return inventories, problems.add(severity, failPrismaSet(inventories, set, message))
@@ -221,9 +238,12 @@ func prismaOutcomeOf(result prismaNormalizationResult) (prismaSetOutcome, string
 }
 
 // configuredPrismaAddresses lists the schema files every configured Prisma glob
-// selects, claim and reference alike, once per population base.
+// selects, claim and reference alike, once per population base, and drops the
+// list of bases that could not be listed.
 //
-// Order is the digest's, so it must not depend on filesystem enumeration.
+// The list is ordered by address key rather than by filesystem enumeration, so
+// the same configuration answers in one order. The order of the parsed set is
+// not taken from here: `distinctPrismaSources` sorts the spellings it keeps.
 func configuredPrismaAddresses(
   config graphConfig,
 ) ([]artifactAddress, graphDiagnostics) {
@@ -231,6 +251,9 @@ func configuredPrismaAddresses(
   return addresses, problems
 }
 
+// configuredPrismaAddressesWithHealth is `configuredPrismaAddresses` together
+// with the bases whose listing failed, so a caller can report the population as
+// failed rather than as healthy and empty.
 func configuredPrismaAddressesWithHealth(
   config graphConfig,
 ) ([]artifactAddress, []populationBase, graphDiagnostics) {
@@ -251,7 +274,7 @@ func configuredPrismaAddressesWithHealth(
       failedBases = append(failedBases, base)
       continue
     }
-    err := filepath.WalkDir(from, func(current string, entry fs.DirEntry, walkErr error) error {
+    err := base.inputs.WalkDir(from, func(current string, entry fs.DirEntry, walkErr error) error {
       if walkErr != nil {
         // The walk root belongs to its population by construction, so a
         // failure to list it is a failure of the population. The relevance test
@@ -367,7 +390,7 @@ func distinctPrismaSources(root string, addresses []artifactAddress) prismaSourc
       continue
     }
     seen[address.Display] = true
-    identity, err := os.Stat(resolveProjectPath(root, address.Display))
+    identity, err := address.Base.inputs.Stat(resolveProjectPath(root, address.Display))
     var host *physicalSchema
     if err == nil {
       for _, file := range files {
@@ -457,7 +480,8 @@ func prismaInventoriesByDisplay(
 //
 // The symbol is `*` rather than `model`, and the difference is load-bearing. A
 // reference reads an inventory problem only when it selects that problem's
-// symbol (`graph.go:169-174`), so a set that failed to parse would look
+// symbol or the problem's symbol is `*` (the reference materializers in
+// `graph.go`), so a set that failed to parse would look
 // problem-free to a reference selecting only columns or relations — which then
 // reports that its globs "materialized no selected evidence units", sending the
 // author to widen a selector when the schema is what could not be read. A
@@ -540,22 +564,42 @@ func couldContainConfiguredPrisma(
   return false
 }
 
+// normalizePrismaSet retains the real Node request/Run/unmarshal boundary.
+// Optional private trace context pairs a caller's already used lookup digest;
+// direct calls allocate an explicitly unpaired observation. Neither context nor
+// observer outcome is added to the request, result or cache identity.
 func normalizePrismaSet(
   root string,
   sources []string,
+  traces ...*evidenceBridgeTrace,
 ) (prismaNormalizationResult, error) {
+  var trace *evidenceBridgeTrace
+  if len(traces) != 0 {
+    trace = traces[0]
+  } else {
+    trace = newEvidenceBridgeTrace("prisma")
+  }
   request, err := json.Marshal(prismaNormalizationRequest{
     Root: root,
     Sets: []prismaSetRequest{{ID: prismaSetID, Files: sources}},
   })
   if err != nil {
+    trace.preparationFailure(err)
     return prismaNormalizationResult{}, err
   }
+  if trace != nil {
+    trace.record("bridge-request", os.Getpid(), map[string]any{
+      "bridge": "prisma", "root": root, "requestId": prismaSetID,
+      "sources": sources, "nativeLookup": trace.nativeLookup,
+    })
+  }
   node := os.Getenv("TTSC_NODE_BINARY")
   if node == "" {
     node, err = exec.LookPath("node")
     if err != nil {
-      return prismaNormalizationResult{}, errors.New("Node.js executable was not found")
+      failure := errors.New("Node.js executable was not found")
+      trace.preparationFailure(failure)
+      return prismaNormalizationResult{}, failure
     }
   }
   ctx, cancel := context.WithTimeout(context.Background(), prismaBridgeTimeout)
@@ -567,20 +611,43 @@ func normalizePrismaSet(
   stderr := &limitedBuffer{Limit: prismaBridgeErrorLimit}
   command.Stdout = stdout
   command.Stderr = stderr
-  if err := command.Run(); err != nil {
-    if ctx.Err() == context.DeadlineExceeded {
+  start := trace.attempt(command)
+  runErr := command.Run()
+  deadlineExceeded := false
+  if runErr != nil {
+    deadlineExceeded = ctx.Err() == context.DeadlineExceeded
+  }
+  end := time.Time{}
+  if trace != nil {
+    end = time.Now().UTC()
+  }
+  if runErr != nil {
+    trace.result(command, stdout, stderr, start, end, runErr, "not-attempted", nil, nil, nil)
+    if deadlineExceeded {
       return prismaNormalizationResult{}, errors.New("Prisma schema loader exceeded its 60 second timeout")
     }
     detail := strings.TrimSpace(stderr.String())
     if detail == "" {
-      detail = err.Error()
+      detail = runErr.Error()
     }
     return prismaNormalizationResult{}, errors.New(detail)
   }
   var result prismaNormalizationResult
   if err := json.Unmarshal(stdout.Bytes(), &result); err != nil {
+    trace.result(command, stdout, stderr, start, end, nil, "failed", err, nil, nil)
     return prismaNormalizationResult{}, errors.New("Prisma schema loader returned invalid JSON: " + err.Error())
   }
+  if trace != nil {
+    documents := make([]string, 0, len(result.Documents))
+    problems := make([]string, 0, len(result.Problems))
+    for _, document := range result.Documents {
+      documents = append(documents, document.ID)
+    }
+    for _, problem := range result.Problems {
+      problems = append(problems, problem.ID)
+    }
+    trace.result(command, stdout, stderr, start, end, nil, "succeeded", nil, documents, problems)
+  }
   return result, nil
 }
 
@@ -645,9 +712,11 @@ func prismaModelUnits(model prismaModel) []*evidenceUnit {
 // so both host a citation here. `//` is discarded by Prisma itself and is the
 // only form that cannot. A top-level unattached `///` run is the one
 // exclusion-only carrier.
-// Every other unusable placement — documenting nothing,
-// documenting something this graph does not address, or burying the tag behind
-// an extra slash — names the move that fixes it.
+//
+// Every other unusable placement, whether a comment trailing code on its own
+// line, documenting nothing, documenting something this graph does not
+// address, or burying the tag behind an extra slash, names the move that fixes
+// it.
 func prismaDeclarationsFromComments(
   comments []prismaCommentRun,
   hosts map[string]*evidenceUnit,
@@ -659,6 +728,19 @@ func prismaDeclarationsFromComments(
   for _, run := range comments {
     severity := levels[run.Path]
     location := run.Path + ":" + decimal(run.Line)
+    if run.Trailing && run.Form != prismaLineComment {
+      // Prisma reads a trailing `///` or `/* */` as documentation of the field
+      // on its own line, so claiming it documents nothing, or that Prisma
+      // discards it, would be false. The graph reads citations only above the
+      // declaration, and this says so.
+      if prismaCommentCarriesTag(run.Body) {
+        problems = problems.add(
+          severity,
+          "Evidence tag at "+location+" sits in a comment that trails code on its own line. A citation is read only from a documentation comment on the lines directly above the model, column, or relation it grounds, so nothing resolves this one. Move the citation directly above that declaration.",
+        )
+      }
+      continue
+    }
     if run.Form != prismaDocComment {
       if prismaCommentCarriesTag(run.Body) {
         problems = problems.add(
@@ -824,8 +906,11 @@ func prismaHostSymbols(host *evidenceUnit) symbolSet {
 // prismaCommentCarriesTag reports whether a comment body opens a citation on
 // any of its lines.
 func prismaCommentCarriesTag(body string) bool {
-  for _, line := range strings.Split(body, "\n") {
-    trimmed := strings.TrimSpace(line)
+  for _, region := range documentationRegions(body, false) {
+    if region.fence {
+      continue
+    }
+    trimmed := region.text
     if _, _, found := declarationLine(trimmed); found {
       return true
     }
@@ -847,8 +932,11 @@ func prismaCommentCarriesTag(body string) bool {
 // an author reading the file sees a citation that does nothing.
 func prismaBuriedTagLines(body string) []int {
   offsets := []int{}
-  for offset, line := range strings.Split(body, "\n") {
-    trimmed := strings.TrimSpace(line)
+  for offset, region := range documentationRegions(body, false) {
+    if region.fence {
+      continue
+    }
+    trimmed := region.text
     if _, _, found := declarationLine(trimmed); found {
       continue
     }
@@ -862,14 +950,11 @@ func prismaBuriedTagLines(body string) []int {
 // prismaBuriedTag reports whether a line would open a citation once its leading
 // comment punctuation is removed.
 //
-// Two shapes bury a tag, and both are one keystroke from a citation that works.
+// An extra slash buries a tag one keystroke from a citation that works.
 // A fourth slash makes `//// @evidence` a doc comment whose text begins with a
-// slash, and a JSDoc-style block hands Prisma its own asterisks as content:
-// measured, `/** @evidence x */` reaches the parser's documentation as
-// `* @evidence x`, and the multi-line form keeps a leading asterisk on every
-// line of it. In both the tag no longer opens its line, so nothing parses it —
-// and until this stripped asterisks too, nothing reported it either. A schema
-// author arriving from JSDoc writes the second shape by habit.
+// slash. The tag no longer opens its line, so the declaration parser cannot
+// read it. JSDoc-style leading asterisks are different: the shared comment
+// parser already strips them, so those block comments host citations normally.
 //
 // Only leading punctuation is stripped, so prose that merely mentions the tag
 // somewhere in a sentence is untouched. Over-reporting an ordinary comment

package/native/prisma_cache.go

@@ -3,7 +3,6 @@ package evidence
 import (
   "crypto/sha256"
   "encoding/hex"
-  "os"
   "strings"
   "sync"
 
@@ -118,17 +117,22 @@ func copyPrismaOutcome(outcome prismaSetOutcome) prismaSetOutcome {
 //
 // The composition is the whole cache key, which is what makes staleness
 // structural rather than improbable: identical bytes in identical order parse
-// to identical models, so a hit can only return what the current schema means.
-// The path is folded in beside each file's own hash, so moving a model between
-// two files of one set — which changes nothing about the bytes as a whole —
-// still changes the key, and so does adding or removing a file.
+// to identical models, so a hit returns what the current schema means to the
+// parser that first read it. The key names no project root and no parser
+// installation, deliberately, so equal schemas share one entry across roots; a
+// resident host serving two projects that resolve different parser builds
+// shares the entry between them. The path is folded in beside each file's own
+// hash, so moving a model between two files of one set — which changes nothing
+// about the bytes as a whole — still changes the key, and so does adding or
+// removing a file. That is the key only: a per-unit digest folds in no path.
 //
 // An unreadable file yields no digest at all, which keeps the set out of the
 // cache and leaves the loader's own diagnostic for it.
-func prismaContentDigest(root string, sources []string) string {
+func prismaContentDigest(root string, sources []string, readers ...evidenceInputReader) string {
+  inputs := inputReader(readers)
   composite := sha256.New()
   for _, source := range sources {
-    content, err := os.ReadFile(resolveProjectPath(root, source))
+    content, err := inputs.ReadFile(resolveProjectPath(root, source))
     if err != nil {
       return ""
     }
@@ -187,7 +191,7 @@ func prismaUnitsFromOutcome(
       prismaNormalizationFailure(outcome.Problem),
     ))
   }
-  locations, comments := locatePrismaDeclarations(root, set.Sources)
+  locations, comments := locatePrismaDeclarations(root, set.Sources, config.inputs)
   fallback := ""
   if len(set.Sources) != 0 {
     fallback = set.Sources[0]