npm - kenaz - Versions diffs - 0.3.21 → 0.3.23 - Mend

kenaz 0.3.21 → 0.3.23

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.

Files changed (481) hide show

package/core/dev/jev-eval/reports/live-smoke-2026-09-20.json DELETED Viewed

@@ -1,616 +0,0 @@
-{
-  "date": "2026-09-20",
-  "model": "jev-1.13.0",
-  "dataset": "dev/jev-eval/fixtures/synthetic.jsonl",
-  "promotion_eligible": false,
-  "installation": {
-    "method": "npx --yes skills add typesafe-ai/skills --skill typesafe-ai --agent codex --yes",
-    "path": ".agents/skills/typesafe-ai/SKILL.md"
-  },
-  "runs": [
-    {
-      "timeout_ms": 10000,
-      "schema_version": 1,
-      "provider": "live",
-      "question_set_version": "jev-eval-v1",
-      "models": [
-        "jev-1.13.0"
-      ],
-      "promotion_eligible": false,
-      "promotion_reason": "Research harness only; metadata is not proof of human review or deployment readiness.",
-      "count": 11,
-      "provenance": {
-        "source": {
-          "synthetic": 11
-        },
-        "annotation": {
-          "agent_proposed": 11
-        },
-        "split": {
-          "calibration": 2,
-          "development": 3,
-          "held_out": 6
-        }
-      },
-      "status_counts": {
-        "ok": 11
-      },
-      "failure_reasons": {},
-      "http_status_counts": {
-        "200": 11
-      },
-      "usage": {
-        "recorded_responses": 11,
-        "input_tokens": 4725,
-        "output_tokens": 323
-      },
-      "metrics": {
-        "correction": {
-          "count": 8,
-          "answered": 8,
-          "abstained_positive": 0,
-          "abstained_negative": 0,
-          "coverage": 1,
-          "confusion": {
-            "tp": 4,
-            "fp": 0,
-            "tn": 4,
-            "fn": 0
-          },
-          "precision": 1,
-          "recall_including_abstentions": 1,
-          "recall_answered_only": 1,
-          "accuracy_including_abstentions": 1,
-          "brier_answered_only": 0.014325000000000001,
-          "brier_count": 8,
-          "threshold": 0.5
-        },
-        "relevance": {
-          "k": 5,
-          "count": 3,
-          "evaluable_queries": 2,
-          "zero_relevance_queries_excluded": 1,
-          "complete_queries": 3,
-          "query_coverage": 1,
-          "candidate_count": 9,
-          "answered_candidates": 9,
-          "incomplete_policy": "Rank answered candidates only; missing candidates are omitted; ideal uses all candidates.",
-          "candidate_coverage": 1,
-          "ndcg_at_k": 1,
-          "ndcg_complete_queries_only": 1,
-          "recall_at_k": 1
-        }
-      },
-      "by_language": {
-        "en": {
-          "correction": {
-            "count": 4,
-            "answered": 4,
-            "abstained_positive": 0,
-            "abstained_negative": 0,
-            "coverage": 1,
-            "confusion": {
-              "tp": 2,
-              "fp": 0,
-              "tn": 2,
-              "fn": 0
-            },
-            "precision": 1,
-            "recall_including_abstentions": 1,
-            "recall_answered_only": 1,
-            "accuracy_including_abstentions": 1,
-            "brier_answered_only": 0.011375000000000003,
-            "brier_count": 4,
-            "threshold": 0.5
-          },
-          "relevance": {
-            "k": 5,
-            "count": 2,
-            "evaluable_queries": 1,
-            "zero_relevance_queries_excluded": 1,
-            "complete_queries": 2,
-            "query_coverage": 1,
-            "candidate_count": 6,
-            "answered_candidates": 6,
-            "incomplete_policy": "Rank answered candidates only; missing candidates are omitted; ideal uses all candidates.",
-            "candidate_coverage": 1,
-            "ndcg_at_k": 1,
-            "ndcg_complete_queries_only": 1,
-            "recall_at_k": 1
-          }
-        },
-        "mixed": {
-          "correction": {
-            "count": 2,
-            "answered": 2,
-            "abstained_positive": 0,
-            "abstained_negative": 0,
-            "coverage": 1,
-            "confusion": {
-              "tp": 1,
-              "fp": 0,
-              "tn": 1,
-              "fn": 0
-            },
-            "precision": 1,
-            "recall_including_abstentions": 1,
-            "recall_answered_only": 1,
-            "accuracy_including_abstentions": 1,
-            "brier_answered_only": 0.032499999999999994,
-            "brier_count": 2,
-            "threshold": 0.5
-          },
-          "relevance": {
-            "k": 5,
-            "count": 0,
-            "evaluable_queries": 0,
-            "zero_relevance_queries_excluded": 0,
-            "complete_queries": 0,
-            "query_coverage": null,
-            "candidate_count": 0,
-            "answered_candidates": 0,
-            "incomplete_policy": "Rank answered candidates only; missing candidates are omitted; ideal uses all candidates.",
-            "candidate_coverage": null,
-            "ndcg_at_k": null,
-            "ndcg_complete_queries_only": null,
-            "recall_at_k": null
-          }
-        },
-        "zh": {
-          "correction": {
-            "count": 2,
-            "answered": 2,
-            "abstained_positive": 0,
-            "abstained_negative": 0,
-            "coverage": 1,
-            "confusion": {
-              "tp": 1,
-              "fp": 0,
-              "tn": 1,
-              "fn": 0
-            },
-            "precision": 1,
-            "recall_including_abstentions": 1,
-            "recall_answered_only": 1,
-            "accuracy_including_abstentions": 1,
-            "brier_answered_only": 0.0020500000000000023,
-            "brier_count": 2,
-            "threshold": 0.5
-          },
-          "relevance": {
-            "k": 5,
-            "count": 1,
-            "evaluable_queries": 1,
-            "zero_relevance_queries_excluded": 0,
-            "complete_queries": 1,
-            "query_coverage": 1,
-            "candidate_count": 3,
-            "answered_candidates": 3,
-            "incomplete_policy": "Rank answered candidates only; missing candidates are omitted; ideal uses all candidates.",
-            "candidate_coverage": 1,
-            "ndcg_at_k": 1,
-            "ndcg_complete_queries_only": 1,
-            "recall_at_k": 1
-          }
-        }
-      },
-      "latency_ms": {
-        "source": "measured_live_end_to_end",
-        "count": 11,
-        "p50": 272.66219999999976,
-        "p95": 735.7556,
-        "p99": 735.7556
-      },
-      "limitations": [
-        "No confidence intervals or promotion gates.",
-        "Exact-content and group leakage checks do not detect paraphrases.",
-        "Brier score covers answered correction items only; consult coverage.",
-        "Rule baseline is lexical and uncalibrated."
-      ]
-    },
-    {
-      "timeout_ms": 800,
-      "schema_version": 1,
-      "provider": "live",
-      "question_set_version": "jev-eval-v1",
-      "models": [
-        "jev-1.13.0"
-      ],
-      "promotion_eligible": false,
-      "promotion_reason": "Research harness only; metadata is not proof of human review or deployment readiness.",
-      "count": 11,
-      "provenance": {
-        "source": {
-          "synthetic": 11
-        },
-        "annotation": {
-          "agent_proposed": 11
-        },
-        "split": {
-          "calibration": 2,
-          "development": 3,
-          "held_out": 6
-        }
-      },
-      "status_counts": {
-        "ok": 11
-      },
-      "failure_reasons": {},
-      "http_status_counts": {
-        "200": 11
-      },
-      "usage": {
-        "recorded_responses": 11,
-        "input_tokens": 4725,
-        "output_tokens": 323
-      },
-      "metrics": {
-        "correction": {
-          "count": 8,
-          "answered": 8,
-          "abstained_positive": 0,
-          "abstained_negative": 0,
-          "coverage": 1,
-          "confusion": {
-            "tp": 4,
-            "fp": 0,
-            "tn": 4,
-            "fn": 0
-          },
-          "precision": 1,
-          "recall_including_abstentions": 1,
-          "recall_answered_only": 1,
-          "accuracy_including_abstentions": 1,
-          "brier_answered_only": 0.013362500000000001,
-          "brier_count": 8,
-          "threshold": 0.5
-        },
-        "relevance": {
-          "k": 5,
-          "count": 3,
-          "evaluable_queries": 2,
-          "zero_relevance_queries_excluded": 1,
-          "complete_queries": 3,
-          "query_coverage": 1,
-          "candidate_count": 9,
-          "answered_candidates": 9,
-          "incomplete_policy": "Rank answered candidates only; missing candidates are omitted; ideal uses all candidates.",
-          "candidate_coverage": 1,
-          "ndcg_at_k": 1,
-          "ndcg_complete_queries_only": 1,
-          "recall_at_k": 1
-        }
-      },
-      "by_language": {
-        "en": {
-          "correction": {
-            "count": 4,
-            "answered": 4,
-            "abstained_positive": 0,
-            "abstained_negative": 0,
-            "coverage": 1,
-            "confusion": {
-              "tp": 2,
-              "fp": 0,
-              "tn": 2,
-              "fn": 0
-            },
-            "precision": 1,
-            "recall_including_abstentions": 1,
-            "recall_answered_only": 1,
-            "accuracy_including_abstentions": 1,
-            "brier_answered_only": 0.011375000000000003,
-            "brier_count": 4,
-            "threshold": 0.5
-          },
-          "relevance": {
-            "k": 5,
-            "count": 2,
-            "evaluable_queries": 1,
-            "zero_relevance_queries_excluded": 1,
-            "complete_queries": 2,
-            "query_coverage": 1,
-            "candidate_count": 6,
-            "answered_candidates": 6,
-            "incomplete_policy": "Rank answered candidates only; missing candidates are omitted; ideal uses all candidates.",
-            "candidate_coverage": 1,
-            "ndcg_at_k": 1,
-            "ndcg_complete_queries_only": 1,
-            "recall_at_k": 1
-          }
-        },
-        "mixed": {
-          "correction": {
-            "count": 2,
-            "answered": 2,
-            "abstained_positive": 0,
-            "abstained_negative": 0,
-            "coverage": 1,
-            "confusion": {
-              "tp": 1,
-              "fp": 0,
-              "tn": 1,
-              "fn": 0
-            },
-            "precision": 1,
-            "recall_including_abstentions": 1,
-            "recall_answered_only": 1,
-            "accuracy_including_abstentions": 1,
-            "brier_answered_only": 0.028099999999999993,
-            "brier_count": 2,
-            "threshold": 0.5
-          },
-          "relevance": {
-            "k": 5,
-            "count": 0,
-            "evaluable_queries": 0,
-            "zero_relevance_queries_excluded": 0,
-            "complete_queries": 0,
-            "query_coverage": null,
-            "candidate_count": 0,
-            "answered_candidates": 0,
-            "incomplete_policy": "Rank answered candidates only; missing candidates are omitted; ideal uses all candidates.",
-            "candidate_coverage": null,
-            "ndcg_at_k": null,
-            "ndcg_complete_queries_only": null,
-            "recall_at_k": null
-          }
-        },
-        "zh": {
-          "correction": {
-            "count": 2,
-            "answered": 2,
-            "abstained_positive": 0,
-            "abstained_negative": 0,
-            "coverage": 1,
-            "confusion": {
-              "tp": 1,
-              "fp": 0,
-              "tn": 1,
-              "fn": 0
-            },
-            "precision": 1,
-            "recall_including_abstentions": 1,
-            "recall_answered_only": 1,
-            "accuracy_including_abstentions": 1,
-            "brier_answered_only": 0.0026000000000000034,
-            "brier_count": 2,
-            "threshold": 0.5
-          },
-          "relevance": {
-            "k": 5,
-            "count": 1,
-            "evaluable_queries": 1,
-            "zero_relevance_queries_excluded": 0,
-            "complete_queries": 1,
-            "query_coverage": 1,
-            "candidate_count": 3,
-            "answered_candidates": 3,
-            "incomplete_policy": "Rank answered candidates only; missing candidates are omitted; ideal uses all candidates.",
-            "candidate_coverage": 1,
-            "ndcg_at_k": 1,
-            "ndcg_complete_queries_only": 1,
-            "recall_at_k": 1
-          }
-        }
-      },
-      "latency_ms": {
-        "source": "measured_live_end_to_end",
-        "count": 11,
-        "p50": 271.8890000000001,
-        "p95": 652.7261,
-        "p99": 652.7261
-      },
-      "limitations": [
-        "No confidence intervals or promotion gates.",
-        "Exact-content and group leakage checks do not detect paraphrases.",
-        "Brier score covers answered correction items only; consult coverage.",
-        "Rule baseline is lexical and uncalibrated."
-      ]
-    }
-  ],
-  "rules_baseline": {
-    "schema_version": 1,
-    "provider": "rules",
-    "question_set_version": "jev-eval-v1",
-    "models": [
-      "rules-v1"
-    ],
-    "promotion_eligible": false,
-    "promotion_reason": "Research harness only; metadata is not proof of human review or deployment readiness.",
-    "count": 11,
-    "provenance": {
-      "source": {
-        "synthetic": 11
-      },
-      "annotation": {
-        "agent_proposed": 11
-      },
-      "split": {
-        "calibration": 2,
-        "development": 3,
-        "held_out": 6
-      }
-    },
-    "status_counts": {
-      "ok": 11
-    },
-    "failure_reasons": {},
-    "http_status_counts": {},
-    "usage": {
-      "recorded_responses": 0,
-      "input_tokens": 0,
-      "output_tokens": 0
-    },
-    "metrics": {
-      "correction": {
-        "count": 8,
-        "answered": 8,
-        "abstained_positive": 0,
-        "abstained_negative": 0,
-        "coverage": 1,
-        "confusion": {
-          "tp": 2,
-          "fp": 2,
-          "tn": 2,
-          "fn": 2
-        },
-        "precision": 0.5,
-        "recall_including_abstentions": 0.5,
-        "recall_answered_only": 0.5,
-        "accuracy_including_abstentions": 0.5,
-        "brier_answered_only": 0.3400000000000001,
-        "brier_count": 8,
-        "threshold": 0.5
-      },
-      "relevance": {
-        "k": 5,
-        "count": 3,
-        "evaluable_queries": 2,
-        "zero_relevance_queries_excluded": 1,
-        "complete_queries": 3,
-        "query_coverage": 1,
-        "candidate_count": 9,
-        "answered_candidates": 9,
-        "incomplete_policy": "Rank answered candidates only; missing candidates are omitted; ideal uses all candidates.",
-        "candidate_coverage": 1,
-        "ndcg_at_k": 0.8221434631015414,
-        "ndcg_complete_queries_only": 0.8221434631015414,
-        "recall_at_k": 1
-      }
-    },
-    "by_language": {
-      "en": {
-        "correction": {
-          "count": 4,
-          "answered": 4,
-          "abstained_positive": 0,
-          "abstained_negative": 0,
-          "coverage": 1,
-          "confusion": {
-            "tp": 1,
-            "fp": 2,
-            "tn": 0,
-            "fn": 1
-          },
-          "precision": 0.3333333333333333,
-          "recall_including_abstentions": 0.5,
-          "recall_answered_only": 0.5,
-          "accuracy_including_abstentions": 0.25,
-          "brier_answered_only": 0.4900000000000001,
-          "brier_count": 4,
-          "threshold": 0.5
-        },
-        "relevance": {
-          "k": 5,
-          "count": 2,
-          "evaluable_queries": 1,
-          "zero_relevance_queries_excluded": 1,
-          "complete_queries": 2,
-          "query_coverage": 1,
-          "candidate_count": 6,
-          "answered_candidates": 6,
-          "incomplete_policy": "Rank answered candidates only; missing candidates are omitted; ideal uses all candidates.",
-          "candidate_coverage": 1,
-          "ndcg_at_k": 0.6442869262030828,
-          "ndcg_complete_queries_only": 0.6442869262030828,
-          "recall_at_k": 1
-        }
-      },
-      "mixed": {
-        "correction": {
-          "count": 2,
-          "answered": 2,
-          "abstained_positive": 0,
-          "abstained_negative": 0,
-          "coverage": 1,
-          "confusion": {
-            "tp": 0,
-            "fp": 0,
-            "tn": 1,
-            "fn": 1
-          },
-          "precision": null,
-          "recall_including_abstentions": 0,
-          "recall_answered_only": 0,
-          "accuracy_including_abstentions": 0.5,
-          "brier_answered_only": 0.3400000000000001,
-          "brier_count": 2,
-          "threshold": 0.5
-        },
-        "relevance": {
-          "k": 5,
-          "count": 0,
-          "evaluable_queries": 0,
-          "zero_relevance_queries_excluded": 0,
-          "complete_queries": 0,
-          "query_coverage": null,
-          "candidate_count": 0,
-          "answered_candidates": 0,
-          "incomplete_policy": "Rank answered candidates only; missing candidates are omitted; ideal uses all candidates.",
-          "candidate_coverage": null,
-          "ndcg_at_k": null,
-          "ndcg_complete_queries_only": null,
-          "recall_at_k": null
-        }
-      },
-      "zh": {
-        "correction": {
-          "count": 2,
-          "answered": 2,
-          "abstained_positive": 0,
-          "abstained_negative": 0,
-          "coverage": 1,
-          "confusion": {
-            "tp": 1,
-            "fp": 0,
-            "tn": 1,
-            "fn": 0
-          },
-          "precision": 1,
-          "recall_including_abstentions": 1,
-          "recall_answered_only": 1,
-          "accuracy_including_abstentions": 1,
-          "brier_answered_only": 0.039999999999999994,
-          "brier_count": 2,
-          "threshold": 0.5
-        },
-        "relevance": {
-          "k": 5,
-          "count": 1,
-          "evaluable_queries": 1,
-          "zero_relevance_queries_excluded": 0,
-          "complete_queries": 1,
-          "query_coverage": 1,
-          "candidate_count": 3,
-          "answered_candidates": 3,
-          "incomplete_policy": "Rank answered candidates only; missing candidates are omitted; ideal uses all candidates.",
-          "candidate_coverage": 1,
-          "ndcg_at_k": 1,
-          "ndcg_complete_queries_only": 1,
-          "recall_at_k": 1
-        }
-      }
-    },
-    "latency_ms": {
-      "source": "measured_local_rules",
-      "count": 11,
-      "p50": 0.009399999999999409,
-      "p95": 0.2564999999999955,
-      "p99": 0.2564999999999955
-    },
-    "limitations": [
-      "No confidence intervals or promotion gates.",
-      "Exact-content and group leakage checks do not detect paraphrases.",
-      "Brier score covers answered correction items only; consult coverage.",
-      "Rule baseline is lexical and uncalibrated."
-    ]
-  },
-  "notes": [
-    "Only synthetic fixture state and candidate text were submitted; credentials were used only in child-process environment and were not persisted in project files.",
-    "Three live batches of 11 requests were executed. The middle 800ms batch stdout was truncated by the tool output limit and is excluded from retained metrics. The two retained runs are the first and third batches.",
-    "Same 11 agent-labeled synthetic rows reused; these are smoke tests, not independent human-reviewed quality validation.",
-    "No provider response cache status was measured; repeated-input caching cannot be excluded. Network, TLS and provider time are combined.",
-    "With n=11, nearest-rank p95 and p99 are both the maximum observation. No SLO or production accuracy claim."
-  ]
-}

package/core/dev/jev-eval/reports/muninn-baseline-2026-09-20.json DELETED Viewed

@@ -1,52 +0,0 @@
-{
-  "schema_version": 1,
-  "date": "2026-09-20",
-  "experiment": "isolated_muninn_prompt_baseline",
-  "promotion_eligible": false,
-  "status": "blocked_by_cli_weekly_limit",
-  "requested_model": "haiku",
-  "observed_model_metadata": null,
-  "source_sha256": "646b55025601441096deec607cc30ce353f777f1dadd81338babc7597abc4dbe",
-  "source_prompt_sha256": "a4e5cbdebed2dc2bc28c415768c433a5ca296f7438def260301c4733ab963f46",
-  "synthetic_correction_items": 8,
-  "live_attempts": {
-    "original_prompt_batches": 2,
-    "diagnostic_prompt_calls": 1,
-    "successful_batches": 0
-  },
-  "first_attempt": {
-    "harness_status": "invalid_or_failed_subprocess",
-    "batch_wall_latency_ms": 1708.1988999999999,
-    "correction_confusion": null,
-    "translation_present": 0
-  },
-  "diagnostic": {
-    "cli_exit_status": 1,
-    "cli_is_error": true,
-    "cli_subtype": "success",
-    "redacted_result": "You've hit your weekly limit · resets 4pm (Asia/Taipei)",
-    "note": "Despite subtype success, process exit and is_error indicate failure. No further retries."
-  },
-  "verification": {
-    "node_tests_passed": 33,
-    "node_tests_failed": 0,
-    "offline_source_contract": "passed",
-    "real_dataset_live_gate": "rejected_synthetic_only_before_transport",
-    "actual_permission_bypass": false
-  },
-  "local_inventory": {
-    "scope": "project .kenaz/transcripts only",
-    "files": 1,
-    "bytes": 1070252,
-    "records": 897,
-    "user_role_records": 22,
-    "independent_sessions_verified": 0,
-    "raw_content_exported": false
-  },
-  "limitations": [
-    "No successful Haiku quality or latency observation, no speedup comparison.",
-    "Existing prompt lacks explicit English target language despite comment contract.",
-    "TypeSafe experiments use different context and tasks; Haiku batch translates and produces seven labels.",
-    "Multiple-session corpus source awaits user selection; existing review seed remains six unreviewed real excerpts."
-  ]
-}