agentskeptic 0.1.1

package/schemas/registry-validation-result.schema.json

@@ -0,0 +1,99 @@
+{
+  "$schema": "https://json-schema.org/draft/2020-12/schema",
+  "$id": "https://agentskeptic.com/schemas/registry-validation-result.schema.json",
+  "title": "RegistryValidationResult",
+  "type": "object",
+  "additionalProperties": false,
+  "required": ["schemaVersion", "valid", "structuralIssues", "resolutionIssues", "resolutionSkipped"],
+  "properties": {
+    "schemaVersion": { "const": 1 },
+    "valid": { "type": "boolean" },
+    "structuralIssues": {
+      "type": "array",
+      "items": { "$ref": "#/$defs/structuralIssue" }
+    },
+    "resolutionIssues": {
+      "type": "array",
+      "items": { "$ref": "#/$defs/resolutionIssue" }
+    },
+    "resolutionSkipped": {
+      "type": "array",
+      "items": { "$ref": "#/$defs/resolutionSkipped" }
+    },
+    "eventLoad": { "$ref": "#/$defs/eventLoad" }
+  },
+  "$defs": {
+    "structuralIssue": {
+      "type": "object",
+      "additionalProperties": false,
+      "required": ["kind", "message"],
+      "properties": {
+        "kind": {
+          "type": "string",
+          "enum": [
+            "json_schema",
+            "duplicate_tool_id",
+            "sql_effects_duplicate_effect_id",
+            "sql_relational_duplicate_check_id"
+          ]
+        },
+        "message": { "type": "string", "minLength": 1 },
+        "instancePath": { "type": "string" },
+        "keyword": { "type": "string" },
+        "toolId": { "type": "string" },
+        "effectId": { "type": "string" },
+        "checkId": { "type": "string" }
+      }
+    },
+    "resolutionIssue": {
+      "type": "object",
+      "additionalProperties": false,
+      "required": ["workflowId", "code", "message", "seq", "toolId"],
+      "properties": {
+        "workflowId": { "type": "string" },
+        "code": { "type": "string" },
+        "message": { "type": "string" },
+        "seq": { "type": ["integer", "null"] },
+        "toolId": { "type": ["string", "null"] }
+      },
+      "allOf": [
+        {
+          "if": { "properties": { "code": { "const": "NO_STEPS_FOR_WORKFLOW" } } },
+          "then": {
+            "properties": {
+              "seq": { "type": "null" },
+              "toolId": { "type": "null" }
+            }
+          },
+          "else": {
+            "properties": {
+              "seq": { "type": "integer" },
+              "toolId": { "type": "string", "minLength": 1 }
+            }
+          }
+        }
+      ]
+    },
+    "resolutionSkipped": {
+      "type": "object",
+      "additionalProperties": false,
+      "required": ["workflowId", "seq", "toolId", "code", "message"],
+      "properties": {
+        "workflowId": { "type": "string" },
+        "seq": { "type": "integer" },
+        "toolId": { "type": "string", "minLength": 1 },
+        "code": { "const": "RETRY_OBSERVATIONS_DIVERGE" },
+        "message": { "type": "string", "minLength": 1 }
+      }
+    },
+    "eventLoad": {
+      "type": "object",
+      "additionalProperties": false,
+      "required": ["workflowId", "malformedEventLineCount"],
+      "properties": {
+        "workflowId": { "type": "string", "minLength": 1 },
+        "malformedEventLineCount": { "type": "integer", "minimum": 0 }
+      }
+    }
+  }
+}

package/schemas/run-comparison-report.schema.json

@@ -0,0 +1,513 @@
+{
+  "$id": "https://agentskeptic.com/schemas/run-comparison-report.schema.json",
+  "$comment": "Structural schema. Behavioral semantics: docs/agentskeptic.md#cross-run-comparison-normative",
+  "title": "RunComparisonReport",
+  "type": "object",
+  "additionalProperties": false,
+  "required": [
+    "schemaVersion",
+    "workflowId",
+    "runs",
+    "perRunActionableFailures",
+    "categoryHistogram",
+    "actionableCategoryRecurrence",
+    "pairwise",
+    "recurrence",
+    "reliabilityAssessment",
+    "compareHighlights"
+  ],
+  "properties": {
+    "schemaVersion": { "const": 4 },
+    "workflowId": { "type": "string" },
+    "runs": {
+      "type": "array",
+      "items": {
+        "type": "object",
+        "additionalProperties": false,
+        "required": ["runIndex", "displayLabel"],
+        "properties": {
+          "runIndex": { "type": "integer", "minimum": 0 },
+          "displayLabel": { "type": "string" }
+        }
+      }
+    },
+    "perRunActionableFailures": {
+      "type": "array",
+      "items": {
+        "type": "object",
+        "additionalProperties": false,
+        "required": ["runIndex", "category", "severity", "recommendedAction", "automationSafe"],
+        "properties": {
+          "runIndex": { "type": "integer", "minimum": 0 },
+          "category": {
+            "oneOf": [
+              {
+                "$ref": "https://agentskeptic.com/schemas/workflow-truth-report.schema.json#/$defs/actionableFailureCategory"
+              },
+              { "const": "complete" }
+            ]
+          },
+          "severity": { "type": "string", "enum": ["high", "medium", "low"] },
+          "recommendedAction": {
+            "$ref": "https://agentskeptic.com/schemas/workflow-truth-report.schema.json#/$defs/recommendedAction"
+          },
+          "automationSafe": { "type": "boolean" }
+        }
+      }
+    },
+    "categoryHistogram": {
+      "type": "array",
+      "items": {
+        "type": "object",
+        "additionalProperties": false,
+        "required": ["category", "count"],
+        "properties": {
+          "category": {
+            "oneOf": [
+              {
+                "$ref": "https://agentskeptic.com/schemas/workflow-truth-report.schema.json#/$defs/actionableFailureCategory"
+              },
+              { "const": "complete" }
+            ]
+          },
+          "count": { "type": "integer", "minimum": 0 }
+        }
+      }
+    },
+    "actionableCategoryRecurrence": {
+      "type": "array",
+      "items": {
+        "type": "object",
+        "additionalProperties": false,
+        "required": [
+          "category",
+          "runIndicesAscending",
+          "runsHitCount",
+          "maxConsecutiveRunStreak"
+        ],
+        "properties": {
+          "category": {
+            "$ref": "https://agentskeptic.com/schemas/workflow-truth-report.schema.json#/$defs/actionableFailureCategory"
+          },
+          "runIndicesAscending": {
+            "type": "array",
+            "items": { "type": "integer", "minimum": 0 }
+          },
+          "runsHitCount": { "type": "integer", "minimum": 1 },
+          "maxConsecutiveRunStreak": { "type": "integer", "minimum": 1 }
+        }
+      }
+    },
+    "pairwise": {
+      "type": "object",
+      "additionalProperties": false,
+      "required": [
+        "priorRunIndex",
+        "currentRunIndex",
+        "runLevel",
+        "ambiguousLogicalKeyResolutions",
+        "bucketA",
+        "bucketB"
+      ],
+      "properties": {
+        "priorRunIndex": { "type": "integer", "minimum": 0 },
+        "currentRunIndex": { "type": "integer", "minimum": 0 },
+        "runLevel": {
+          "type": "object",
+          "additionalProperties": false,
+          "required": ["introducedRunLevelCodes", "resolvedRunLevelCodes"],
+          "properties": {
+            "introducedRunLevelCodes": {
+              "type": "array",
+              "items": { "type": "string" }
+            },
+            "resolvedRunLevelCodes": {
+              "type": "array",
+              "items": { "type": "string" }
+            }
+          }
+        },
+        "ambiguousLogicalKeyResolutions": {
+          "type": "array",
+          "items": {
+            "type": "object",
+            "additionalProperties": false,
+            "required": ["logicalStepKey", "chosenSeq", "droppedSeq"],
+            "properties": {
+              "logicalStepKey": { "type": "string" },
+              "chosenSeq": { "type": "integer", "minimum": 0 },
+              "droppedSeq": { "type": "integer", "minimum": 0 }
+            }
+          }
+        },
+        "bucketA": {
+          "type": "array",
+          "items": { "$ref": "#/$defs/bucketAEntry" }
+        },
+        "bucketB": {
+          "type": "object",
+          "additionalProperties": false,
+          "required": [
+            "introducedFailureSignatures",
+            "resolvedFailureSignatures",
+            "unchangedFailureInstanceCounts"
+          ],
+          "properties": {
+            "introducedFailureSignatures": {
+              "type": "array",
+              "items": {
+                "type": "object",
+                "additionalProperties": false,
+                "required": ["signature", "count"],
+                "properties": {
+                  "signature": { "type": "string" },
+                  "count": { "type": "integer", "minimum": 1 }
+                }
+              }
+            },
+            "resolvedFailureSignatures": {
+              "type": "array",
+              "items": {
+                "type": "object",
+                "additionalProperties": false,
+                "required": ["signature", "count"],
+                "properties": {
+                  "signature": { "type": "string" },
+                  "count": { "type": "integer", "minimum": 1 }
+                }
+              }
+            },
+            "unchangedFailureInstanceCounts": {
+              "type": "array",
+              "items": {
+                "type": "object",
+                "additionalProperties": false,
+                "required": ["signature", "matchedCount"],
+                "properties": {
+                  "signature": { "type": "string" },
+                  "matchedCount": { "type": "integer", "minimum": 1 }
+                }
+              }
+            }
+          }
+        }
+      }
+    },
+    "reliabilityAssessment": {
+      "type": "object",
+      "additionalProperties": false,
+      "required": [
+        "windowTrend",
+        "pairwiseTrend",
+        "recurrenceBurden",
+        "headlineVerdict",
+        "headlineRationale"
+      ],
+      "properties": {
+        "windowTrend": {
+          "type": "string",
+          "enum": ["improving", "worsening", "unchanged", "mixed"]
+        },
+        "pairwiseTrend": {
+          "type": "string",
+          "enum": ["improving", "worsening", "unchanged", "mixed"]
+        },
+        "recurrenceBurden": {
+          "type": "object",
+          "additionalProperties": false,
+          "required": ["patternCount", "maxRunsHitCount", "crossRunFailure", "rationale"],
+          "properties": {
+            "patternCount": { "type": "integer", "minimum": 0 },
+            "maxRunsHitCount": { "type": "integer", "minimum": 0 },
+            "crossRunFailure": { "type": "boolean" },
+            "rationale": { "type": "string" }
+          }
+        },
+        "headlineVerdict": {
+          "type": "string",
+          "enum": ["improving", "worsening", "unchanged", "mixed"]
+        },
+        "headlineRationale": { "type": "string" }
+      }
+    },
+    "compareHighlights": {
+      "type": "object",
+      "additionalProperties": false,
+      "required": [
+        "introducedLogicalStepKeys",
+        "resolvedLogicalStepKeys",
+        "bothFailingChurn",
+        "bucketBIntroducedSignatures",
+        "bucketBResolvedSignatures",
+        "recurringSignatures"
+      ],
+      "properties": {
+        "introducedLogicalStepKeys": {
+          "type": "array",
+          "items": { "type": "string" }
+        },
+        "resolvedLogicalStepKeys": {
+          "type": "array",
+          "items": { "type": "string" }
+        },
+        "bothFailingChurn": {
+          "type": "array",
+          "items": {
+            "type": "object",
+            "additionalProperties": false,
+            "required": [
+              "logicalStepKey",
+              "introducedStepReasonCodes",
+              "resolvedStepReasonCodes"
+            ],
+            "properties": {
+              "logicalStepKey": { "type": "string" },
+              "introducedStepReasonCodes": {
+                "type": "array",
+                "items": { "type": "string" }
+              },
+              "resolvedStepReasonCodes": {
+                "type": "array",
+                "items": { "type": "string" }
+              }
+            }
+          }
+        },
+        "bucketBIntroducedSignatures": {
+          "type": "array",
+          "items": {
+            "type": "object",
+            "additionalProperties": false,
+            "required": ["signature", "count"],
+            "properties": {
+              "signature": { "type": "string" },
+              "count": { "type": "integer", "minimum": 1 }
+            }
+          }
+        },
+        "bucketBResolvedSignatures": {
+          "type": "array",
+          "items": {
+            "type": "object",
+            "additionalProperties": false,
+            "required": ["signature", "count"],
+            "properties": {
+              "signature": { "type": "string" },
+              "count": { "type": "integer", "minimum": 1 }
+            }
+          }
+        },
+        "recurringSignatures": {
+          "type": "array",
+          "items": { "type": "string" }
+        }
+      }
+    },
+    "recurrence": {
+      "type": "object",
+      "additionalProperties": false,
+      "required": ["patterns"],
+      "properties": {
+        "patterns": {
+          "type": "array",
+          "items": {
+            "type": "object",
+            "additionalProperties": false,
+            "required": ["signature", "runIndices", "runsHitCount", "exemplars"],
+            "properties": {
+              "signature": { "type": "string" },
+              "runIndices": {
+                "type": "array",
+                "items": { "type": "integer", "minimum": 0 }
+              },
+              "runsHitCount": { "type": "integer", "minimum": 2 },
+              "exemplars": {
+                "type": "array",
+                "items": {
+                  "type": "object",
+                  "additionalProperties": false,
+                  "required": ["runIndex", "seq", "toolId"],
+                  "properties": {
+                    "runIndex": { "type": "integer", "minimum": 0 },
+                    "seq": { "type": "integer", "minimum": 0 },
+                    "toolId": { "type": "string" }
+                  }
+                }
+              }
+            }
+          }
+        }
+      }
+    }
+  },
+  "$defs": {
+    "bucketAEntry": {
+      "oneOf": [
+        {
+          "type": "object",
+          "additionalProperties": false,
+          "required": [
+            "kind",
+            "logicalStepKey",
+            "seqPrior",
+            "seqCurrent",
+            "toolIdPrior",
+            "toolIdCurrent"
+          ],
+          "properties": {
+            "kind": { "const": "unchangedOk" },
+            "logicalStepKey": { "type": "string" },
+            "seqPrior": { "type": "integer", "minimum": 0 },
+            "seqCurrent": { "type": "integer", "minimum": 0 },
+            "toolIdPrior": { "type": "string" },
+            "toolIdCurrent": { "type": "string" }
+          }
+        },
+        {
+          "type": "object",
+          "additionalProperties": false,
+          "required": [
+            "kind",
+            "logicalStepKey",
+            "seqPrior",
+            "seqCurrent",
+            "toolIdPrior",
+            "toolIdCurrent"
+          ],
+          "properties": {
+            "kind": { "const": "introducedFailure" },
+            "logicalStepKey": { "type": "string" },
+            "seqPrior": { "type": "integer", "minimum": 0 },
+            "seqCurrent": { "type": "integer", "minimum": 0 },
+            "toolIdPrior": { "type": "string" },
+            "toolIdCurrent": { "type": "string" }
+          }
+        },
+        {
+          "type": "object",
+          "additionalProperties": false,
+          "required": [
+            "kind",
+            "logicalStepKey",
+            "seqPrior",
+            "seqCurrent",
+            "toolIdPrior",
+            "toolIdCurrent"
+          ],
+          "properties": {
+            "kind": { "const": "resolvedFailure" },
+            "logicalStepKey": { "type": "string" },
+            "seqPrior": { "type": "integer", "minimum": 0 },
+            "seqCurrent": { "type": "integer", "minimum": 0 },
+            "toolIdPrior": { "type": "string" },
+            "toolIdCurrent": { "type": "string" }
+          }
+        },
+        {
+          "type": "object",
+          "additionalProperties": false,
+          "required": [
+            "kind",
+            "logicalStepKey",
+            "seqPrior",
+            "toolIdPrior",
+            "priorWasFailing"
+          ],
+          "properties": {
+            "kind": { "const": "structuralRemoval" },
+            "logicalStepKey": { "type": "string" },
+            "seqPrior": { "type": "integer", "minimum": 0 },
+            "toolIdPrior": { "type": "string" },
+            "priorWasFailing": { "type": "boolean" }
+          }
+        },
+        {
+          "type": "object",
+          "additionalProperties": false,
+          "required": ["kind", "logicalStepKey", "seqCurrent", "toolIdCurrent", "currentIsFailing"],
+          "properties": {
+            "kind": { "const": "structuralAddition" },
+            "logicalStepKey": { "type": "string" },
+            "seqCurrent": { "type": "integer", "minimum": 0 },
+            "toolIdCurrent": { "type": "string" },
+            "currentIsFailing": { "type": "boolean" }
+          }
+        },
+        {
+          "type": "object",
+          "additionalProperties": false,
+          "required": [
+            "kind",
+            "logicalStepKey",
+            "seqPrior",
+            "seqCurrent",
+            "toolIdPrior",
+            "toolIdCurrent",
+            "toolIdChanged",
+            "introducedStepReasonCodes",
+            "resolvedStepReasonCodes",
+            "effects"
+          ],
+          "properties": {
+            "kind": { "const": "bothFailing" },
+            "logicalStepKey": { "type": "string" },
+            "seqPrior": { "type": "integer", "minimum": 0 },
+            "seqCurrent": { "type": "integer", "minimum": 0 },
+            "toolIdPrior": { "type": "string" },
+            "toolIdCurrent": { "type": "string" },
+            "toolIdChanged": { "type": "boolean" },
+            "introducedStepReasonCodes": {
+              "type": "array",
+              "items": { "type": "string" }
+            },
+            "resolvedStepReasonCodes": {
+              "type": "array",
+              "items": { "type": "string" }
+            },
+            "effects": {
+              "type": "array",
+              "items": {
+                "type": "object",
+                "additionalProperties": false,
+                "required": [
+                  "effectId",
+                  "kind",
+                  "introducedReasonCodes",
+                  "resolvedReasonCodes",
+                  "statusPrior",
+                  "statusCurrent"
+                ],
+                "properties": {
+                  "effectId": { "type": "string" },
+                  "kind": {
+                    "type": "string",
+                    "enum": [
+                      "introducedFailure",
+                      "resolvedFailure",
+                      "bothFailing",
+                      "unchangedOk"
+                    ]
+                  },
+                  "introducedReasonCodes": {
+                    "type": "array",
+                    "items": { "type": "string" }
+                  },
+                  "resolvedReasonCodes": {
+                    "type": "array",
+                    "items": { "type": "string" }
+                  },
+                  "statusPrior": {
+                    "type": ["string", "null"]
+                  },
+                  "statusCurrent": {
+                    "type": ["string", "null"]
+                  }
+                }
+              }
+            }
+          }
+        }
+      ]
+    }
+  }
+}

package/schemas/tools-registry-export.schema.json

@@ -0,0 +1,9 @@
+{
+  "$id": "https://agentskeptic.com/schemas/tools-registry-export.schema.json",
+  "title": "ToolsRegistryExportArray",
+  "description": "Quick Verify --export-registry file: same items as tools-registry but allows empty array.",
+  "type": "array",
+  "items": {
+    "$ref": "https://agentskeptic.com/schemas/tools-registry.schema.json#/items"
+  }
+}