@zalom/plastic 2.0.0-alpha.14 → 2.0.0-alpha.15

package/hooks/message-display

@@ -107,4 +107,28 @@ fi
 
 [ "$handoff" = 1 ] || exit 0
 
+# 331a1 (D1): chunk 0 is about to hand off to Ruby, which takes on the
+# order of 150 ms to boot before it ever writes SCREEN or NOSCREEN. Claude
+# Code fires the per-chunk hook PROCESSES CONCURRENTLY (see the header
+# comment above), so a later chunk landing inside that window used to find
+# no message directory at all, fail this script's own cheap shape test,
+# and pass through as raw Markdown - measured at 15 to 21 chunks of every
+# screen, against a stream-realistic 5 ms stagger. Staking a PENDING
+# marker here, the instant the handoff gate above passes, closes that
+# window down to microseconds: a later chunk's own "does the directory
+# exist" check (below, unchanged) now finds it almost at once, and
+# MessageDisplay polls for the real decision instead of judging the
+# chunk's own shape. mkdir is the one non-builtin on this whole path - it
+# forks, unlike everything else in this file - but it runs ONLY here,
+# after the handoff gate, so the ordinary (non-candidate) chunk that is
+# the overwhelming common case still forks nothing (matrix L8). A failed
+# mkdir or printf (an unwritable tmp root, a race with another process) is
+# silently swallowed: it must never change this script's own exit status
+# or skip the Ruby handoff below - Ruby's own polling already fails open
+# (D5) when no decision ever appears.
+if [ "$is_index_zero" = 1 ]; then
+  mkdir -p "$MSGDIR" 2>/dev/null
+  printf '' > "$MSGDIR/PENDING" 2>/dev/null
+fi
+
 printf '%s' "$INPUT" | env -u RUBYOPT ruby "$SCRIPT_DIR/../scripts/hook-message-display"

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@zalom/plastic",
-  "version": "2.0.0-alpha.14",
+  "version": "2.0.0-alpha.15",
   "description": "Intent-driven idea development system for AI coding agents",
   "type": "module",
   "bin": {

package/scripts/lib/hook_replay.rb

@@ -56,6 +56,89 @@ module HookReplay
     end
   end
 
+  # replay_concurrent (intent 331a1) - streams `text` through `hook_path` the
+  # way `replay` does, but fires every chunk in its OWN thread, staggered by
+  # `gap_ms` (plus up to half a gap of jitter when `jitter` is true) rather
+  # than run sequentially. This is what reproduces the decision-race defect
+  # 331a1 fixes: Claude Code fires the per-chunk hook processes CONCURRENTLY
+  # in production, and `replay`'s strictly-sequential default never puts two
+  # chunks in flight at once, so it could never have reproduced the race in
+  # the first place.
+  #
+  # `gap_ms: 5` plus jitter is the default on purpose, not "fire everything
+  # at once": firing all 335 chunks of the live session capture with no
+  # stagger at all takes about 8 s of wall clock on this 8-core machine,
+  # because EACH chunk boots its own real Ruby process and the completions
+  # cluster at the tail once every core is saturated - a load real streaming
+  # never produces (a real stream delivers a chunk every few tens of
+  # milliseconds, one at a time). One Ruby process per streamed chunk is the
+  # actual throughput ceiling here, not something this method works around.
+  #
+  # `replay`'s own signature and sequential default are UNCHANGED by this
+  # method's existence (`scripts/doctor.rb:2571` calls `replay` directly and
+  # must keep working exactly as it does today) - this is a sibling method
+  # in the same module, never a replacement.
+  #
+  # Returns the SAME result shape `replay` returns (one Hash per chunk, keys
+  # index/exitstatus/stdout/stderr/final), ordered by index regardless of
+  # the order the threads actually finish in.
+  def replay_concurrent(hook_path:, tmp_root:, text:, chunk: 40, session_id: "s-replay",
+                         message_id: "replay", env: {}, gap_ms: 5, jitter: true)
+    chunks = text.scan(/.{1,#{chunk}}/m)
+    chunks = [""] if chunks.empty?
+    full_env = { "PLASTIC_TMP" => tmp_root }.merge(env)
+    gap = gap_ms / 1000.0
+
+    results = Array.new(chunks.length)
+    threads = chunks.each_with_index.map do |delta, i|
+      payload = {
+        "session_id" => session_id, "message_id" => message_id, "index" => i,
+        "final" => i == chunks.length - 1, "delta" => delta, "cwd" => tmp_root,
+        "hook_event_name" => "MessageDisplay",
+      }
+      Thread.new do
+        begin
+          delay = i * gap
+          delay += (rand * gap / 2.0) if jitter
+          sleep(delay)
+          out, err, exitstatus = run_one(hook_path, payload, full_env, tmp_root, nil)
+          results[i] = { index: i, exitstatus: exitstatus, stdout: out, stderr: err, final: payload["final"] }
+        rescue StandardError => e
+          # A raise inside a thread body is invisible until join, and an
+          # unrescued one aborts `threads.each(&:join)` at the first dead
+          # thread: every later thread is then never joined and outlives the
+          # call, racing whatever the caller does next (typically removing
+          # the very tmp root those threads are still writing under). Report
+          # the failure as this chunk's own result instead, so the array is
+          # always complete, every thread is always joined, and a replay
+          # tells its caller what went wrong rather than throwing at it.
+          results[i] = { index: i, exitstatus: nil, stdout: "", stderr: e.message,
+                         final: payload["final"] }
+        end
+      end
+    end
+    threads.each { |thread| thread.join }
+    results
+  end
+
+  # Indices of the chunks that reached the terminal as raw Markdown: a
+  # non-final chunk that emitted nothing at all, after the engaging chunk.
+  # A chunk "passed through" when its stdout is empty; a chunk was
+  # "blanked" (correctly buffered, not shown raw) when its stdout contains
+  # `"displayContent":""`. Only chunks with an index greater than the
+  # engaging chunk's own index count - the engaging chunk is the first one
+  # (at or after `start_index`) whose stdout is non-empty, and chunks
+  # before it already reached the terminal live, verbatim, through the
+  # ordinary passthrough path (they were never candidates for buffering at
+  # all, so an empty stdout from one of them is not this defect).
+  def passthrough_indices(outs, start_index: 0)
+    engaging = outs.find { |o| o[:index] >= start_index && !o[:stdout].to_s.empty? }
+    return [] unless engaging
+
+    outs.select { |o| o[:index] > engaging[:index] && o[:final] != true && o[:stdout].to_s.empty? }
+        .map { |o| o[:index] }
+  end
+
   def run_one(hook_path, payload, full_env, tmp_root, timeout)
     return capture(hook_path, payload, full_env) unless timeout
 

package/scripts/lib/message_display.rb

@@ -60,6 +60,35 @@ require_relative "screen_paint"
 # (D4): a lone fence in the engaging chunk's own prefix, and a lone closing
 # fence right after the painted region in `finalize`. Neither ever reaches
 # back into an earlier, already-displayed chunk.
+#
+# Intent 331a1: the decision marker (D1-D3). 331a's own comment above already
+# names the concurrency; what it did not close is chunk 0's own boot time.
+# Chunk 0's Ruby process takes on the order of 150 ms to boot before it ever
+# writes SCREEN or NOSCREEN - long enough, under a fast real stream, for a
+# dozen or more later chunks to be judged with nothing on disk at all, so
+# every one of them fell back to the cheap shape test and, being ordinary
+# non-table prose, passed straight through plain. The bash launcher (hooks/
+# message-display) now stakes a PENDING file with builtins the instant
+# chunk 0 is handed off, before Ruby ever starts, so a later chunk finds the
+# message directory within microseconds instead of after Ruby's own boot.
+# While PENDING exists, a later chunk polls for the real decision WHATEVER
+# ITS OWN SHAPE looks like - `maybe_screen?` is not consulted at all, because
+# a decision is certainly coming, and the cheap shape gate exists only for
+# the "nothing at all exists yet, is a wait even worth paying for" case,
+# which no longer applies once something IS on disk. A PENDING whose mtime
+# is already older than THIS chunk's own poll budget reads as NOSCREEN (D2,
+# fail open): chunk 0 must have died or hung, and waiting out a whole budget
+# for a decision that is provably not coming would only delay every chunk
+# behind it. That staleness check runs ONCE, before any polling, since a
+# file's mtime never changes while this process looks at it. The poll
+# budget itself scales with the chunk's own index (`budget_ms`, D3): base
+# wait_ms plus index_wait_ms per index, capped at max_wait_ms, so the final
+# chunk of a long streamed message (335 chunks, in the live capture that
+# reproduced this) is allowed to wait for a decision that is certainly on
+# its way, while chunk 1 of an ordinary short message still fails open
+# quickly. Chunk 0 removes PENDING the moment it writes SCREEN or NOSCREEN
+# (`write_screen`/`write_noscreen`), on both paths, so "a decision already
+# exists" and "PENDING is still there" are never both true for long.
 class MessageDisplay
   # 317a (A4): engagement is grammar, not identity - any screen-family
   # opener engages, with NO intent-id resolution (the roster and delay
@@ -76,8 +105,13 @@ class MessageDisplay
   BUFFER_MAX_AGE_SECONDS = 3600
   SCREEN_FILE = "SCREEN"
   NOSCREEN_FILE = "NOSCREEN"
+  # 331a1 (D1): staked by the bash launcher, with builtins, the instant
+  # chunk 0 is handed off - before Ruby ever boots. Replaced by SCREEN or
+  # NOSCREEN (D2), never read by this class as a decision in its own right.
+  PENDING_FILE = "PENDING"
 
   def initialize(tmp_root:, plastic_home:, color:, now:, wait_ms: 300, poll_ms: 20,
+                 index_wait_ms: 20, max_wait_ms: 2000,
                  sleeper: ->(seconds) { sleep(seconds) })
     @tmp_root = tmp_root
     @plastic_home = plastic_home
@@ -85,6 +119,8 @@ class MessageDisplay
     @now = now
     @wait_ms = wait_ms
     @poll_ms = poll_ms
+    @index_wait_ms = index_wait_ms
+    @max_wait_ms = max_wait_ms
     @sleeper = sleeper
   end
 
@@ -132,6 +168,13 @@ class MessageDisplay
     File.join(buffer_path(tmp_root: tmp_root, session_id: session_id, message_id: message_id), NOSCREEN_FILE)
   end
 
+  # 331a1 (matrix L1): the bash launcher (hooks/message-display) and this
+  # class must agree, byte for byte, on where PENDING lives - the same
+  # contract `buffer_path` already carries for SCREEN/NOSCREEN (matrix 40).
+  def self.pending_path(tmp_root:, session_id:, message_id:)
+    File.join(buffer_path(tmp_root: tmp_root, session_id: session_id, message_id: message_id), PENDING_FILE)
+  end
+
   private
 
   # Chunk 0 decides, synchronously, before anything else touches this
@@ -159,7 +202,7 @@ class MessageDisplay
     split = split_at_opener(delta)
     return engage(dir, index, split, final) if split
 
-    decision = wait_for_decision(dir, gate_delta: final ? nil : delta)
+    decision = wait_for_decision(dir, gate_delta: final ? nil : delta, index: index)
 
     return nil unless decision == :screen
 
@@ -212,17 +255,34 @@ class MessageDisplay
     lines[0...-1].join
   end
 
-  # Checks for an existing decision first (free) and only pays the cheap
-  # shape test, then the bounded poll, when neither SCREEN nor NOSCREEN is
-  # there yet. `gate_delta: nil` (the final chunk) skips the shape test
-  # entirely and always polls for the decision.
-  def wait_for_decision(dir, gate_delta:)
+  # Checks for an existing decision first (free). Then, 331a1 (D1/D2), the
+  # whole fix: when PENDING exists, a decision is certainly coming, so this
+  # chunk polls for it WHATEVER ITS OWN SHAPE looks like - `maybe_screen?`
+  # is never even consulted on this branch - unless PENDING is already
+  # stale (older than this chunk's own budget), which reads as NOSCREEN at
+  # once, fail open, without ever polling. Only when there is no PENDING
+  # AT ALL (chunk 0 has not even been handed off to the bash launcher yet,
+  # or this replay never wrote one) does today's original behavior apply:
+  # the cheap shape test gates whether a bounded poll is worth paying for.
+  # `gate_delta: nil` (the final chunk) skips that shape test entirely and
+  # always polls.
+  def wait_for_decision(dir, gate_delta:, index:)
     decision = read_decision_now(dir)
     return decision if decision
 
+    if pending?(dir)
+      return :noscreen if pending_stale?(dir, index)
+
+      return poll_for_decision(dir, index)
+    end
+
     return :timeout if gate_delta && !maybe_screen?(gate_delta)
 
-    max_polls_for_budget.times do
+    poll_for_decision(dir, index)
+  end
+
+  def poll_for_decision(dir, index)
+    max_polls_for_budget(index).times do
       @sleeper.call(@poll_ms / 1000.0)
       decision = read_decision_now(dir)
       return decision if decision
@@ -238,6 +298,24 @@ class MessageDisplay
     nil
   end
 
+  def pending?(dir)
+    File.exist?(File.join(dir, PENDING_FILE))
+  end
+
+  # Checked ONCE, before any polling - a file's mtime never changes while
+  # this process is looking at it, so re-checking inside the poll loop
+  # would only ever repeat the same answer. Any error reading the mtime
+  # (a race where PENDING vanished between `pending?` and here, most
+  # likely because the real decision just landed) is NOT staleness: it
+  # falls through to the ordinary poll, which will pick up that decision
+  # on its very next read.
+  def pending_stale?(dir, index)
+    age_ms = (@now.to_f - File.mtime(File.join(dir, PENDING_FILE)).to_f) * 1000
+    age_ms > budget_ms(index)
+  rescue StandardError
+    false
+  end
+
   # 331a (M5a): the start index crosses process boundaries through SCREEN's
   # own content, never in-memory state - the final chunk is routinely a
   # SEPARATE process from the one that engaged. An empty or missing file
@@ -288,21 +366,34 @@ class MessageDisplay
     end
   end
 
+  # 331a1 (D3): index-scaled too, same as the decision poll - the final
+  # chunk of a long streamed message (335 chunks, in the live capture that
+  # reproduced this) must be allowed to wait long enough for the earlier
+  # chunk files to land, not just the base wait_ms an ordinary short
+  # message gets by with.
   def wait_for_chunk_files(dir, start_index, index)
     return if index <= start_index
 
     needed = (start_index...index).map(&:to_s)
-    max_polls_for_budget.times do
+    max_polls_for_budget(index).times do
       return if needed.all? { |n| File.exist?(File.join(dir, n)) }
 
       @sleeper.call(@poll_ms / 1000.0)
     end
   end
 
-  def max_polls_for_budget
+  # 331a1 (D3): base wait_ms plus index_wait_ms per chunk index, capped at
+  # max_wait_ms - a chunk deep into a long streamed message is certainly
+  # going to see its decision eventually, so it is allowed to wait longer
+  # than chunk 1 of an ordinary short message.
+  def budget_ms(index)
+    [@wait_ms + @index_wait_ms * index.to_i, @max_wait_ms].min
+  end
+
+  def max_polls_for_budget(index)
     return 0 unless @poll_ms.to_f.positive?
 
-    (@wait_ms / @poll_ms.to_f).ceil
+    (budget_ms(index) / @poll_ms.to_f).ceil
   end
 
   # Whatever chunk files exist FROM THE START INDEX onward, in index order,
@@ -363,12 +454,18 @@ class MessageDisplay
   # separate process, in production) knows where to start waiting and
   # splicing, and so `finalize_final` never touches chunks that were passed
   # through untouched before engagement.
+  # 331a1 (D2): the decision REPLACES PENDING, on this path too, whichever
+  # chunk turns out to be the one that engages.
   def write_screen(dir, index)
     atomic_write(File.join(dir, SCREEN_FILE), "#{index}\n")
+    remove_pending(dir)
   end
 
+  # 331a1 (D2): same replacement on the NOSCREEN path, so a later chunk
+  # never finds both PENDING and NOSCREEN and has to choose between them.
   def write_noscreen(dir)
     atomic_write(File.join(dir, NOSCREEN_FILE), "")
+    remove_pending(dir)
   end
 
   # 331a (D2/D6): NOSCREEN is no longer a final answer - a later chunk that
@@ -378,6 +475,15 @@ class MessageDisplay
     FileUtils.rm_f(File.join(dir, NOSCREEN_FILE))
   end
 
+  # 331a1 (D2): removal must never raise - a decision was already written
+  # successfully by the time this runs, and a stray filesystem error here
+  # must never turn a successful decision into an unhandled exception.
+  def remove_pending(dir)
+    FileUtils.rm_f(File.join(dir, PENDING_FILE))
+  rescue StandardError
+    nil
+  end
+
   def atomic_write(path, content)
     FileUtils.mkdir_p(File.dirname(path))
     tmp_path = "#{path}.tmp#{Process.pid}-#{rand(1_000_000)}"

package/scripts/lib/report_screen.rb

@@ -197,9 +197,25 @@ module ReportScreen
     budget = limit - (4 + 3 * (ncols - 1)) - overage
     widths = ScreenPaint.shrink_column_widths(widths, budget, bar_columns: bar_column, floors: floors)
 
+    # 331f1a (D1/D2, plan-review ruling): a separator row passes through byte-identical
+    # whenever its OWN unfitted input already fits the bound - rebuilding it from the
+    # shrunk widths (with the `[w, 3].max` floor below) is what made it assemble wider
+    # than any data row in the first place, landing it as the only row the backstop ever
+    # cut (or, when the rebuilt form happened to still fit, wider than its own "---"
+    # input, which D1 forbids just as much). Only when even the unmodified input cannot
+    # fit does the old rebuild-and-backstop path apply - the bound wins there, which is
+    # exactly what test_fit_screen_backstops_an_unshrinkable_row and
+    # test_unshrinkable_data_table_is_still_bounded pin. A blank "| | | |" scaffold
+    # reaching this branch is classified as a separator by the same regex, so it gets
+    # the identical pass-through rule.
     fitted_rows = raw_rows.each_with_index.map do |cells, ri|
       if is_sep[ri]
-        "| #{widths.map { |w| "-" * [w, 3].max }.join(" | ")} |"
+        original = rows[ri]
+        if ScreenPaint.display_columns(original) <= limit
+          original
+        else
+          "| #{widths.map { |w| "-" * [w, 3].max }.join(" | ")} |"
+        end
       else
         rendered = cells.each_with_index.map do |c, ci|
           next c.to_s.strip if ci >= ncols
@@ -215,7 +231,8 @@ module ReportScreen
     # and the assembled row can still be over the limit; truncate the whole row on a word
     # boundary rather than let it survive past 115 - a data table's separator row included
     # (test_fit_screen_backstops_an_unshrinkable_row), unlike the field-table fitter's own
-    # separator, which always passes through untouched (W2).
+    # separator, which always passes through untouched (W2). A separator already passed
+    # through byte-identical above never trips this (it already fits by construction).
     fitted_rows.map! { |r| ScreenPaint.display_columns(r) > limit ? truncate_on_word_boundary(r, limit) : r }
 
     "#{fitted_rows.join("\n")}\n"