@mitre/hdf-schema 3.7.2 → 3.8.0-next.42532249979.1

package/dist/ts/hdf.ts

@@ -19,10 +19,10 @@ export interface HDFResults {
      */
     derivation?: Derivation;
     /**
-     * Reserved for tool-specific data not defined in the HDF standard. Use this to preserve
-     * original tool output, auxiliary data, or custom metadata.
+     * Producer data outside the HDF model, and the pre-converted source artifact(s) these
+     * results were converted from.
      */
-    extensions?: { [key: string]: any };
+    extensions?: Extensions;
     /**
      * Optional references to external artifacts (CTI/STIX, BOMs, advisories, runbooks, or any
      * URI-addressable artifact) relevant to this assessment as a whole. Inert context; see
@@ -101,9 +101,10 @@ export interface EvaluatedBaseline {
      */
     description?: string;
     /**
-     * Reserved for tool-specific baseline metadata not defined in the HDF standard.
+     * Producer data outside the HDF model that belongs to this baseline rather than to the
+     * document, and the pre-converted source artifact(s) this baseline was built from.
      */
-    extensions?: { [key: string]: any };
+    extensions?: Extensions;
     /**
      * A set of descriptions for the requirement groups.
      */
@@ -242,6 +243,52 @@ export interface Dependency {
     [property: string]: any;
 }
 
+/**
+ * Carrier for data outside the HDF model: `passthrough` for producer data HDF does not model,
+ * `rawSourceArtifacts` for the pre-converted input(s). CLOSED: these two members are the whole of
+ * it. A key HDF does not define goes inside `passthrough` — that is what `passthrough` is for — so
+ * `extensions` never becomes a second, unnamed schema that consumers must reverse-engineer from the
+ * producers that happen to write it. Closed with `additionalProperties` rather than the house
+ * `unevaluatedProperties` because this definition composes nothing, and the shipped Go validator is
+ * a draft-07 engine that does not implement `unevaluatedProperties` — a closure only one of the two
+ * validators enforces is not a closure. Declared on every HDF document type that carries a producer
+ * envelope (`generator` / `integrity`), and on `Evaluated_Baseline`, so one rule governs both
+ * sites.
+ */
+export interface Extensions {
+    /**
+     * Catch-all for producer data that HDF does not model — the role heimdall2's
+     * `passthrough.auxiliary_data` played: fields a converter deliberately carried but had no
+     * home for.
+     *
+     * Deliberately permissive: unenumerated properties are accepted by design, with no shape
+     * imposed on them. HDF neither reads nor interprets what is here. It is the ONLY permissive
+     * surface in `extensions` — everything HDF does not define belongs here rather than beside
+     * it.
+     *
+     * This is NOT where the pre-converted source artifact goes. v2 put it here only because
+     * nothing else had room; `rawSourceArtifacts` is its home now, and it gives the bytes a
+     * checksum, a media type and a reference contract that an untyped object cannot.
+     *
+     * For data that belongs to one requirement rather than to the document, use that object's
+     * own fields — a document-level bag cannot say which requirement it describes.
+     */
+    passthrough?: { [key: string]: any };
+    /**
+     * The input(s) this document was converted from, verbatim.
+     *
+     * An array because a conversion can consume more than one artifact — the OSCAL profile path
+     * takes a profile and a catalog. Every other converter takes one input, and an array of one
+     * costs nothing.
+     *
+     * Populating this is required of ingest converters by policy, enforced by the converter
+     * contract and test harness rather than by a schema `required`: a tool that emits HDF
+     * natively has no source artifact, and requiring the field would make valid native HDF
+     * invalid.
+     */
+    rawSourceArtifacts?: ExternalReference[];
+}
+
 /**
  * A generalized reference to any external artifact by identity and/or location, modeled on the STIX
  * 2.1 `external_references` common property. Purpose-agnostic: cite a CVE, an ATT&CK technique, a
@@ -249,8 +296,11 @@ export interface Dependency {
  * HDF reference category. `sourceName` names the system; `externalId` cites by id within that
  * system; `href` locates it. `sourceName` + `externalId` is equivalently a URN
  * (`urn:<sourceName>:<externalId>`, RFC 8141), so the by-identity and by-location forms stay
- * interconvertible. A reference is inert context: it overrides nothing, so it carries only
- * lightweight optional `addedBy`/`addedAt` attribution, never override machinery.
+ * interconvertible. An entry may also carry the artifact itself rather than point at it: `document`
+ * for a JSON object, `content` + `encoding` for anything else. Embedding satisfies the at-least-one
+ * rule on its own, so an artifact with no meaningful public location needs no synthetic `href`. A
+ * reference is inert context: it overrides nothing, so it carries only lightweight optional
+ * `addedBy`/`addedAt` attribution, never override machinery.
  */
 export interface ExternalReference {
     /**
@@ -263,18 +313,36 @@ export interface ExternalReference {
      */
     addedBy?: Identity;
     /**
-     * Integrity hash of the referenced artifact. Reuses the HDF `Checksum` primitive (not STIX
-     * `hashes`). Meaningful only with a retrievable `href`.
+     * Integrity hash of the artifact. Reuses the HDF `Checksum` primitive (not STIX `hashes`).
+     * Applies both to an artifact retrievable at `href` and to one embedded here in `content`
+     * or `document` — beside embedded bytes it is what makes the copy verifiable rather than
+     * merely present.
      */
     checksum?: Checksum;
+    /**
+     * Optional lossless embedded copy of the artifact's bytes as a string, for artifacts
+     * `document` cannot hold: XML, CSV, NDJSON, plain text, a JSON array, or bytes that are not
+     * valid UTF-8 text at all.
+     *
+     * `encoding` states how to read it back and is required alongside it. Set `mediaType` so a
+     * consumer knows what the bytes are, and `checksum` so the copy can be verified against the
+     * source.
+     *
+     * Carried verbatim — never reformatted, re-indented, or re-serialized.
+     */
+    content?: string;
     /**
      * Human-readable description of what is referenced. Satisfies the at-least-one rule on its
      * own when neither id nor href is available.
      */
     description?: string;
     /**
-     * Optional lossless embedded copy of the referenced artifact — the raw STIX object, EPSS
-     * record, advisory, or annotation payload — preserved verbatim.
+     * Optional lossless embedded copy of the referenced artifact as a JSON object — the raw
+     * STIX object, EPSS record, advisory, or annotation payload — preserved verbatim.
+     *
+     * JSON objects only. An artifact that is not a JSON object (XML, CSV, NDJSON, plain text, a
+     * JSON array, or bytes that are not text at all) embeds through `content` + `encoding`
+     * instead.
      *
      * Composes with `href`/`externalId` (which point at the artifact) and `checksum` (which
      * ties the copy to the source): a single entry can both point and embed.
@@ -283,6 +351,15 @@ export interface ExternalReference {
      * fields — so this does not duplicate the source ontology (e.g. STIX rides here unchanged).
      */
     document?: { [key: string]: any };
+    /**
+     * How to read `content` back into the artifact's bytes. 'utf-8' for text (XML, CSV, NDJSON,
+     * plain text, JavaScript-wrapped JSON); 'base64' for anything that is not valid UTF-8
+     * text.
+     *
+     * Required whenever `content` is present, and meaningless without it — an embedded copy
+     * whose encoding a consumer has to guess is not lossless.
+     */
+    encoding?: ExternalReferenceEncoding;
     /**
      * Identifier of the artifact within `sourceName` (e.g., 'CVE-2021-44228' for source 'cve',
      * 'T1059' for 'mitre-att&ck'). Cite by id without needing a URL. Together with `sourceName`
@@ -291,8 +368,8 @@ export interface ExternalReference {
     externalId?: string;
     /**
      * Location of the artifact. `uri-reference` (not `uri`) so a bare internal
-     * `#fragment`/`#uuid` reference is expressible alongside absolute URLs.
-     * `checksum`/`mediaType` apply only to a retrievable `href`.
+     * `#fragment`/`#uuid` reference is expressible alongside absolute URLs. Set it only when
+     * the location is meaningful to a reader — never a producer's local filesystem path.
      */
     href?: string;
     /**
@@ -309,8 +386,9 @@ export interface ExternalReference {
      */
     kind?: string;
     /**
-     * IANA media type of the referenced artifact when retrievable (RFC 6838), e.g.,
-     * 'application/json'. Meaningful only with `href`.
+     * IANA media type of the artifact (RFC 6838), e.g., 'application/json', 'application/xml',
+     * 'text/csv'. Applies both to an artifact retrievable at `href` and to one embedded here in
+     * `content` or `document`.
      */
     mediaType?: string;
     /**
@@ -393,8 +471,9 @@ export interface Checksum {
 }
 
 /**
- * Supported cryptographic hash algorithms for checksums and integrity verification. blake3 covers
- * container-image and other artifact digests that use it.
+ * Cryptographic hash algorithms a checksum or integrity record may name. blake3 covers
+ * container-image and other artifact digests that use it. HDF tooling verifies every algorithm in
+ * this enum, so any of them may be recorded where the checksum needs to be verifiable.
  */
 export enum HashAlgorithm {
     Blake3 = "blake3",
@@ -403,6 +482,21 @@ export enum HashAlgorithm {
     Sha512 = "sha512",
 }
 
+/**
+ * How to read `content` back into the artifact's bytes. 'utf-8' for text (XML, CSV, NDJSON,
+ * plain text, JavaScript-wrapped JSON); 'base64' for anything that is not valid UTF-8
+ * text.
+ *
+ * Required whenever `content` is present, and meaningless without it — an embedded copy
+ * whose encoding a consumer has to guess is not lossless.
+ *
+ * How to read `content`: 'utf-8' for text, 'base64' for bytes that are not valid UTF-8 text.
+ */
+export enum ExternalReferenceEncoding {
+    Base64 = "base64",
+    UTF8 = "utf-8",
+}
+
 /**
  * Describes a group of requirements, such as those defined in a single file.
  */
@@ -1325,6 +1419,13 @@ export interface RequirementResult {
      * explain why a test was not applicable or not reviewed.
      */
     message?: string;
+    /**
+     * The source tool's own record for this result, carrying every field the record had.
+     * Complete, not necessarily byte-exact: see Raw_Source_Record.content for the producer and
+     * consumer obligations, and extensions.rawSourceArtifacts[] when byte-exact carriage is
+     * required.
+     */
+    rawSourceRecord?: RawSourceRecord;
     /**
      * The resource used in the test. Example: 'file', 'command', 'service'.
      */
@@ -1348,6 +1449,67 @@ export interface RequirementResult {
     [property: string]: any;
 }
 
+/**
+ * The source tool's own record for this result — the one entry in the converter's input that
+ * produced it, complete though not necessarily byte-for-byte (see `content`). SCOPED TO THE RESULT
+ * DELIBERATELY. For a vulnerability scanner the native record is one (vulnerability, package) pair,
+ * which is exactly one result: a CVE reported against two packages yields two entries whose records
+ * differ only in the package they matched. Held on the requirement instead, those two records would
+ * collide, and merging entries that share an id would have to discard one — so the result is not
+ * merely an available home but the correct one. Distinct from `extensions.rawSourceArtifacts[]`,
+ * which carries whole input artifacts: this carries the one record inside one of them, and
+ * `pointer` ties the two together. A consumer that wants the whole input reads the artifact; one
+ * that wants provenance for a single finding reads this. Distinct from `code`, which means the code
+ * that ran to evaluate the requirement — an InSpec control's source, or a check reference. A
+ * serialized finding is not code, and putting one there overloaded a field whose meaning came from
+ * InSpec. CLOSED, and deliberately not a passthrough bag: three required members and an optional
+ * pointer are the whole of it. Producer data that HDF does not model belongs in
+ * `extensions.passthrough`, which is document-scoped and permissive by design; this field is
+ * neither. Closed with `additionalProperties` rather than the house `unevaluatedProperties` because
+ * this definition composes nothing, and the shipped Go validator is a draft-07 engine that does not
+ * implement `unevaluatedProperties` — a closure only one of the two validators enforces is not a
+ * closure.
+ */
+export interface RawSourceRecord {
+    /**
+     * The source tool's own record for this result, carrying every field the record had.
+     *
+     * COMPLETE, NOT NECESSARILY BYTE-EXACT. A producer SHOULD write the record's bytes
+     * unchanged where it can. A producer that cannot — notably one implemented in two languages
+     * whose JSON serializers disagree on key order, HTML escaping and number rendering, and
+     * which therefore MUST canonicalize to stay byte-identical with its twin — MAY write a
+     * canonically serialized form instead. Such a form preserves every field and value but not
+     * the original key order, insignificant whitespace, or duplicate keys. A consumer MUST
+     * therefore compare records by parsing them, not by comparing bytes, and MUST NOT checksum
+     * this field and expect the result to match a checksum of the input.
+     *
+     * For carriage that IS byte-exact, use `extensions.rawSourceArtifacts[]`, which forbids a
+     * parsed `document` precisely so the bytes and their checksum survive; `pointer` ties this
+     * record to its position within that artifact.
+     */
+    content: string;
+    /**
+     * How to read `content`: 'utf-8' for text, 'base64' for bytes that are not valid UTF-8 text.
+     */
+    encoding: ExternalReferenceEncoding;
+    /**
+     * The record's IANA media type — the type of this record, which for a fragment of a larger
+     * artifact may be narrower than the artifact's own. Example: 'application/json' for one
+     * match object out of a JSON report.
+     */
+    mediaType: string;
+    /**
+     * Where this record sits inside the artifact it came from, as a JSON Pointer (RFC 6901) for
+     * JSON sources or an XPath for XML ones. Example: '/matches/3'.
+     *
+     * Optional, and the link that makes carriage verifiable rather than merely present: with it
+     * a consumer can confirm the record really is the subtree of
+     * `extensions.rawSourceArtifacts[]` it claims to be. Omit it when the source's shape gives
+     * a record no addressable location.
+     */
+    pointer?: string;
+}
+
 /**
  * Severity rating for a requirement. Typically derived from the numeric impact score.
  */
@@ -2372,6 +2534,11 @@ export interface HDFBaseline {
      * The set of dependencies this baseline depends on.
      */
     depends?: Dependency[];
+    /**
+     * Producer data outside the HDF model, and the pre-converted source artifact(s) this
+     * baseline was converted from.
+     */
+    extensions?: Extensions;
     /**
      * The tool that generated this file.
      */
@@ -2576,9 +2743,10 @@ export interface HDFComparison {
      */
     drift?: RequirementDiff[];
     /**
-     * Reserved for tool-specific data not defined in the HDF standard.
+     * Producer data outside the HDF model, and the pre-converted source artifact(s) this
+     * comparison was derived from.
      */
-    extensions?: { [key: string]: any };
+    extensions?: Extensions;
     /**
      * Optional references to external artifacts relevant to this comparison (CTI/STIX,
      * advisories, or any URI-addressable artifact). Inert context; see External_Reference.
@@ -3365,6 +3533,11 @@ export interface HDFSystem {
      * Description of the system's purpose and mission.
      */
     description?: string;
+    /**
+     * Producer data outside the HDF model, and the pre-converted source artifact(s) this system
+     * document was converted from.
+     */
+    extensions?: Extensions;
     /**
      * Optional references to external artifacts describing this system's threat environment or
      * context (CTI/STIX, BOMs, advisories, or any URI-addressable artifact). Inert context; see
@@ -3553,6 +3726,11 @@ export interface HDFPlan {
      * Description of the plan's purpose and scope.
      */
     description?: string;
+    /**
+     * Producer data outside the HDF model, and the pre-converted source artifact(s) this plan
+     * was converted from.
+     */
+    extensions?: Extensions;
     /**
      * Optional references to external artifacts relevant to this assessment plan (CTI/STIX,
      * advisories, methodology docs, or any URI-addressable artifact). Inert context; see
@@ -3713,6 +3891,12 @@ export interface HDFAmendments {
      * Description of the amendments' purpose and scope.
      */
     description?: string;
+    /**
+     * Producer data outside the HDF model, and the pre-converted source artifact(s) these
+     * amendments were converted from — the VEX family and oscal-poam-to-hdf all emit this
+     * document type.
+     */
+    extensions?: Extensions;
     /**
      * Information about the tool that generated this document.
      */
@@ -3890,6 +4074,11 @@ export interface HDFEvidencePackage {
      * Description of the evidence package's purpose and scope.
      */
     description?: string;
+    /**
+     * Producer data outside the HDF model, and the pre-converted source artifact(s) this
+     * package was assembled from.
+     */
+    extensions?: Extensions;
     /**
      * References to external native-format evidence (log/telemetry corpora and other artifacts)
      * carried by URI + integrity hash + format discriminator, without recreating the data
@@ -4024,7 +4213,17 @@ export interface ContentReference {
      */
     type: ContentType;
     /**
-     * URI to the document. Can be a relative path or absolute URL.
+     * A uri-reference whose DECODED form is a path relative to the package document's own
+     * directory — that is what the reference MEANS, independently of any one consumer.
+     * Percent-escapes are part of the encoding, not the name: `my%20scans/q4.json` and `my
+     * scans/q4.json` name the same file, and the escaped spelling is the one this field's
+     * format admits. A consumer decodes before it resolves, never after, because decoding can
+     * produce a parent traversal that containment must then see. It is never fetched: there is
+     * no network retrieval anywhere in evidence handling. `uri-reference` admits shapes other
+     * than a relative path and they stay schema-valid; how a consumer resolves one, and what it
+     * does when it cannot, is that consumer's own contract. An artifact deliberately held
+     * elsewhere belongs in externalEvidence[], which references it by URI and checksum without
+     * retrieving it.
      */
     uri: string;
     [property: string]: any;

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@mitre/hdf-schema",
-  "version": "3.7.2",
+  "version": "3.8.0-next.42532249979.1",
   "description": "JSON schemas and multi-language type definitions for Heimdall Data Format (HDF)",
   "publishConfig": {
     "access": "public"
@@ -63,9 +63,9 @@
   "author": "MITRE Corporation",
   "license": "Apache-2.0",
   "dependencies": {
+    "@mitre/hdf-utilities": "3.8.0-next.42532249979.1",
     "ajv": "^8.17.0",
-    "ajv-formats": "^3.0.0",
-    "@mitre/hdf-utilities": "^3.7.2"
+    "ajv-formats": "^3.0.0"
   },
   "devDependencies": {
     "@hyperjump/json-schema": "^1.17.2",