@zanii/train 0.2.1 → 0.2.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +4 -0
- package/dist/index.d.ts +1 -1
- package/dist/index.d.ts.map +1 -1
- package/dist/index.js +1 -0
- package/dist/index.js.map +1 -1
- package/dist/provenance.d.ts +180 -0
- package/dist/provenance.d.ts.map +1 -0
- package/dist/provenance.js +114 -0
- package/dist/provenance.js.map +1 -0
- package/package.json +2 -2
- package/src/index.ts +1 -0
- package/src/provenance.ts +157 -0
package/README.md
CHANGED
|
@@ -43,3 +43,7 @@ absence proof is only as honest as the manifest is complete — pair it with an
|
|
|
43
43
|
enclave (`@zanii/attest`: the published image hash is the only code that touched the
|
|
44
44
|
data) so "absent from the manifest" also means "absent from the run." The manifest
|
|
45
45
|
commits refs, never content.
|
|
46
|
+
|
|
47
|
+
## Changelog
|
|
48
|
+
|
|
49
|
+
- **0.2.2** — Dataset cards and an AI Act training-content summary bound to the manifest, opt-out and eval-contamination checks with absence proofs, a licence ledger, synthetic-data depth, run attestations, fine-tune manifests, royalties by inclusion, poisoning forensics.
|
package/dist/index.d.ts
CHANGED
|
@@ -55,5 +55,5 @@ export declare function proveMembership(m: TrainingManifest, refs: string[], ref
|
|
|
55
55
|
export declare function verifyAbsence(proof: AbsenceProof | undefined, m: TrainingManifest): boolean;
|
|
56
56
|
/** Verify a membership proof against a manifest. */
|
|
57
57
|
export declare function verifyMembership(proof: MembershipProof | undefined, m: TrainingManifest): boolean;
|
|
58
|
-
export
|
|
58
|
+
export * from './provenance.js';
|
|
59
59
|
//# sourceMappingURL=index.d.ts.map
|
package/dist/index.d.ts.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AA2BA,yFAAyF;AACzF,UAAU,IAAI;IACZ,EAAE,EAAE,MAAM,CAAC;IACX,EAAE,EAAE,MAAM,CAAC;CACZ;AAED,MAAM,WAAW,gBAAgB;IAC/B,CAAC,EAAE,CAAC,CAAC;IACL,IAAI,EAAE,mBAAmB,CAAC;IAC1B,sCAAsC;IACtC,MAAM,EAAE,MAAM,CAAC;IACf,qCAAqC;IACrC,IAAI,EAAE,MAAM,CAAC;IACb,qEAAqE;IACrE,IAAI,EAAE,MAAM,CAAC;IACb,UAAU,EAAE,MAAM,CAAC;CACpB;AA8BD,+FAA+F;AAC/F,wBAAgB,aAAa,CAAC,MAAM,EAAE;IAAE,KAAK,EAAE,MAAM,CAAC;IAAC,IAAI,EAAE,MAAM,EAAE,CAAC;IAAC,SAAS,EAAE,MAAM,CAAA;CAAE,GAAG,gBAAgB,CAa5G;AAED,wBAAgB,YAAY,CAAC,CAAC,EAAE,gBAAgB,GAAG,MAAM,CAExD;AAED,0FAA0F;AAC1F,wBAAgB,eAAe,CAAC,CAAC,EAAE,gBAAgB,GAAG;IAAE,MAAM,EAAE,MAAM,CAAC;IAAC,OAAO,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAA;CAAE,CAEzG;AAQD,MAAM,WAAW,YAAY;IAC3B,CAAC,EAAE,CAAC,CAAC;IACL,IAAI,EAAE,kBAAkB,CAAC;IACzB,MAAM,EAAE,MAAM,CAAC;IACf,6BAA6B;IAC7B,GAAG,EAAE,MAAM,CAAC;IACZ,wDAAwD;IACxD,GAAG,EAAE,IAAI,CAAC;IACV,KAAK,EAAE,MAAM,CAAC;IACd,IAAI,EAAE,MAAM,CAAC;IACb,IAAI,EAAE,MAAM,EAAE,CAAC;CAChB;AAED,MAAM,WAAW,eAAe;IAC9B,CAAC,EAAE,CAAC,CAAC;IACL,IAAI,EAAE,qBAAqB,CAAC;IAC5B,MAAM,EAAE,MAAM,CAAC;IACf,GAAG,EAAE,MAAM,CAAC;IACZ,KAAK,EAAE,MAAM,CAAC;IACd,IAAI,EAAE,MAAM,CAAC;IACb,IAAI,EAAE,MAAM,EAAE,CAAC;CAChB;AAED,yFAAyF;AACzF,wBAAgB,YAAY,CAAC,CAAC,EAAE,gBAAgB,EAAE,IAAI,EAAE,MAAM,EAAE,EAAE,GAAG,EAAE,MAAM,GAAG,YAAY,CAS3F;AAED,iFAAiF;AACjF,wBAAgB,eAAe,CAAC,CAAC,EAAE,gBAAgB,EAAE,IAAI,EAAE,MAAM,EAAE,EAAE,GAAG,EAAE,MAAM,GAAG,eAAe,CAMjG;AAED,6FAA6F;AAC7F,wBAAgB,aAAa,CAAC,KAAK,EAAE,YAAY,GAAG,SAAS,EAAE,CAAC,EAAE,gBAAgB,GAAG,OAAO,CAY3F;AAED,oDAAoD;AACpD,wBAAgB,gBAAgB,CAAC,KAAK,EAAE,eAAe,GAAG,SAAS,EAAE,CAAC,EAAE,gBAAgB,GAAG,OAAO,CASjG"}
|
|
1
|
+
{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AA2BA,yFAAyF;AACzF,UAAU,IAAI;IACZ,EAAE,EAAE,MAAM,CAAC;IACX,EAAE,EAAE,MAAM,CAAC;CACZ;AAED,MAAM,WAAW,gBAAgB;IAC/B,CAAC,EAAE,CAAC,CAAC;IACL,IAAI,EAAE,mBAAmB,CAAC;IAC1B,sCAAsC;IACtC,MAAM,EAAE,MAAM,CAAC;IACf,qCAAqC;IACrC,IAAI,EAAE,MAAM,CAAC;IACb,qEAAqE;IACrE,IAAI,EAAE,MAAM,CAAC;IACb,UAAU,EAAE,MAAM,CAAC;CACpB;AA8BD,+FAA+F;AAC/F,wBAAgB,aAAa,CAAC,MAAM,EAAE;IAAE,KAAK,EAAE,MAAM,CAAC;IAAC,IAAI,EAAE,MAAM,EAAE,CAAC;IAAC,SAAS,EAAE,MAAM,CAAA;CAAE,GAAG,gBAAgB,CAa5G;AAED,wBAAgB,YAAY,CAAC,CAAC,EAAE,gBAAgB,GAAG,MAAM,CAExD;AAED,0FAA0F;AAC1F,wBAAgB,eAAe,CAAC,CAAC,EAAE,gBAAgB,GAAG;IAAE,MAAM,EAAE,MAAM,CAAC;IAAC,OAAO,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAA;CAAE,CAEzG;AAQD,MAAM,WAAW,YAAY;IAC3B,CAAC,EAAE,CAAC,CAAC;IACL,IAAI,EAAE,kBAAkB,CAAC;IACzB,MAAM,EAAE,MAAM,CAAC;IACf,6BAA6B;IAC7B,GAAG,EAAE,MAAM,CAAC;IACZ,wDAAwD;IACxD,GAAG,EAAE,IAAI,CAAC;IACV,KAAK,EAAE,MAAM,CAAC;IACd,IAAI,EAAE,MAAM,CAAC;IACb,IAAI,EAAE,MAAM,EAAE,CAAC;CAChB;AAED,MAAM,WAAW,eAAe;IAC9B,CAAC,EAAE,CAAC,CAAC;IACL,IAAI,EAAE,qBAAqB,CAAC;IAC5B,MAAM,EAAE,MAAM,CAAC;IACf,GAAG,EAAE,MAAM,CAAC;IACZ,KAAK,EAAE,MAAM,CAAC;IACd,IAAI,EAAE,MAAM,CAAC;IACb,IAAI,EAAE,MAAM,EAAE,CAAC;CAChB;AAED,yFAAyF;AACzF,wBAAgB,YAAY,CAAC,CAAC,EAAE,gBAAgB,EAAE,IAAI,EAAE,MAAM,EAAE,EAAE,GAAG,EAAE,MAAM,GAAG,YAAY,CAS3F;AAED,iFAAiF;AACjF,wBAAgB,eAAe,CAAC,CAAC,EAAE,gBAAgB,EAAE,IAAI,EAAE,MAAM,EAAE,EAAE,GAAG,EAAE,MAAM,GAAG,eAAe,CAMjG;AAED,6FAA6F;AAC7F,wBAAgB,aAAa,CAAC,KAAK,EAAE,YAAY,GAAG,SAAS,EAAE,CAAC,EAAE,gBAAgB,GAAG,OAAO,CAY3F;AAED,oDAAoD;AACpD,wBAAgB,gBAAgB,CAAC,KAAK,EAAE,eAAe,GAAG,SAAS,EAAE,CAAC,EAAE,gBAAgB,GAAG,OAAO,CASjG;AACD,cAAc,iBAAiB,CAAC"}
|
package/dist/index.js
CHANGED
package/dist/index.js.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"index.js","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;GAuBG;AACH,OAAO,EAAE,UAAU,EAAE,UAAU,EAAE,MAAM,qBAAqB,CAAC;AAC7D,OAAO,EAAE,cAAc,EAAE,cAAc,EAAE,OAAO,EAAE,QAAQ,EAAE,UAAU,EAAE,eAAe,EAAE,MAAM,aAAa,CAAC;AAoB7G,yFAAyF;AACzF,gFAAgF;AAChF,SAAS,SAAS,CAAC,CAAS;IAC1B,IAAI,OAAO,CAAC,KAAK,QAAQ,IAAI,CAAC,CAAC,MAAM,KAAK,CAAC;QAAE,MAAM,IAAI,KAAK,CAAC,gCAAgC,CAAC,CAAC;IAC/F,KAAK,MAAM,CAAC,IAAI,CAAC,EAAE,CAAC;QAClB,IAAI,CAAC,CAAC,WAAW,CAAC,CAAC,CAAE,IAAI,MAAM;YAAE,MAAM,IAAI,KAAK,CAAC,4EAA4E,IAAI,CAAC,SAAS,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC;IACpJ,CAAC;AACH,CAAC;AAED,0FAA0F;AAC1F,SAAS,OAAO,CAAC,IAAc;IAC7B,KAAK,MAAM,CAAC,IAAI,IAAI;QAAE,SAAS,CAAC,CAAC,CAAC,CAAC;IACnC,MAAM,MAAM,GAAG,CAAC,GAAG,IAAI,GAAG,CAAC,IAAI,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC;IACzC,qFAAqF;IACrF,IAAI,MAAM,CAAC,MAAM,KAAK,CAAC;QAAE,OAAO,CAAC,EAAE,EAAE,EAAE,EAAE,EAAE,EAAE,EAAE,GAAG,EAAE,CAAC,CAAC;IACtD,MAAM,KAAK,GAAW,EAAE,CAAC;IACzB,wEAAwE;IACxE,KAAK,CAAC,IAAI,CAAC,EAAE,EAAE,EAAE,EAAE,EAAE,EAAE,EAAE,MAAM,CAAC,CAAC,CAAE,EAAE,CAAC,CAAC;IACvC,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,MAAM,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE,CAAC;QACvC,KAAK,CAAC,IAAI,CAAC,EAAE,EAAE,EAAE,MAAM,CAAC,CAAC,CAAE,EAAE,EAAE,EAAE,MAAM,CAAC,CAAC,CAAE,EAAE,CAAC,CAAC,CAAC,sCAAsC;QACtF,MAAM,IAAI,GAAG,CAAC,GAAG,CAAC,GAAG,MAAM,CAAC,MAAM,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,GAAG,CAAC,CAAE,CAAC,CAAC,CAAC,GAAG,CAAC;QAC1D,KAAK,CAAC,IAAI,CAAC,EAAE,EAAE,EAAE,MAAM,CAAC,CAAC,CAAE,EAAE,EAAE,EAAE,IAAI,EAAE,CAAC,CAAC,CAAC,6BAA6B;IACzE,CAAC;IACD,OAAO,KAAK,CAAC;AACf,CAAC;AAED,MAAM,QAAQ,GAAG,CAAC,CAAO,EAAc,EAAE,CAAC,QAAQ,CAAC,cAAc,CAAC,CAAuC,CAAC,CAAC,CAAC;AAE5G,+FAA+F;AAC/F,MAAM,UAAU,aAAa,CAAC,MAA4D;IACxF,IAAI,CAAC,MAAM,CAAC,KAAK;QAAE,MAAM,IAAI,KAAK,CAAC,mBAAmB,CAAC,CAAC;IACxD,IAAI,MAAM,CAAC,KAAK,CAAC,IAAI,CAAC,KAAK,CAAC,MAAM,CAAC,SAAS,CAAC,CAAC;QAAE,MAAM,IAAI,KAAK,CAAC,oCAAoC,CAAC,CAAC;IACtG,MAAM,KAAK,GAAG,OAAO,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC;IACnC,MAAM,IAAI,GAAG,UAAU,CAAC,UAAU,CAAC,KAAK,CAAC,GAAG,CAAC,QAAQ,CAAC,CAAC,CAAC,CAAC;IACzD,OAAO;QACL,CAAC,EAAE,CAAC;QACJ,IAAI,EAAE,mBAAmB;QACzB,MAAM,EAAE,MAAM,CAAC,KAAK;QACpB,IAAI,EAAE,IAAI,GAAG,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC,IAAI;QAC/B,IAAI,EAAE,UAAU,IAAI,EAAE;QACtB,UAAU,EAAE,MAAM,CAAC,SAAS;KAC7B,CAAC;AACJ,CAAC;AAED,MAAM,UAAU,YAAY,CAAC,CAAmB;IAC9C,OAAO,OAAO,CAAC,CAAuC,CAAC,CAAC;AAC1D,CAAC;AAED,0FAA0F;AAC1F,MAAM,UAAU,eAAe,CAAC,CAAmB;IACjD,OAAO,EAAE,MAAM,EAAE,cAAc,EAAE,OAAO,EAAE,EAAE,QAAQ,EAAE,CAAC,EAAE,aAAa,EAAE,YAAY,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC;AAC9F,CAAC;AAED,4EAA4E;AAC5E,gFAAgF;AAChF,MAAM,aAAa,GAAG,CAAC,KAAa,EAAE,MAAoB,EAAY,EAAE,CAAC,cAAc,CAAC,KAAK,EAAE,MAAM,CAAC,CAAC,GAAG,CAAC,UAAU,CAAC,CAAC;AACvH,MAAM,UAAU,GAAG,CAAC,IAAgB,EAAE,KAAa,EAAE,IAAY,EAAE,IAAc,EAAE,IAAgB,EAAW,EAAE,CAC9G,eAAe,CAAC,IAAI,EAAE,KAAK,EAAE,IAAI,EAAE,IAAI,CAAC,GAAG,CAAC,UAAU,CAAC,EAAE,IAAI,CAAC,CAAC;AAyBjE,yFAAyF;AACzF,MAAM,UAAU,YAAY,CAAC,CAAmB,EAAE,IAAc,EAAE,GAAW;IAC3E,SAAS,CAAC,GAAG,CAAC,CAAC;IACf,MAAM,MAAM,GAAG,CAAC,GAAG,IAAI,GAAG,CAAC,IAAI,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC;IACzC,IAAI,MAAM,CAAC,QAAQ,CAAC,GAAG,CAAC;QAAE,MAAM,IAAI,KAAK,CAAC,uCAAuC,CAAC,CAAC;IACnF,MAAM,KAAK,GAAG,OAAO,CAAC,IAAI,CAAC,CAAC;IAC5B,MAAM,MAAM,GAAG,KAAK,CAAC,GAAG,CAAC,QAAQ,CAAC,CAAC;IACnC,MAAM,EAAE,GAAG,KAAK,CAAC,SAAS,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,EAAE,GAAG,GAAG,IAAI,GAAG,GAAG,CAAC,CAAC,EAAE,IAAI,CAAC,CAAC,CAAC,CAAC,EAAE,KAAK,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC;IAChF,IAAI,EAAE,GAAG,CAAC;QAAE,MAAM,IAAI,KAAK,CAAC,sCAAsC,CAAC,CAAC;IACpE,OAAO,EAAE,CAAC,EAAE,CAAC,EAAE,IAAI,EAAE,kBAAkB,EAAE,MAAM,EAAE,CAAC,CAAC,MAAM,EAAE,GAAG,EAAE,GAAG,EAAE,KAAK,CAAC,EAAE,CAAE,EAAE,KAAK,EAAE,EAAE,EAAE,IAAI,EAAE,MAAM,CAAC,MAAM,EAAE,IAAI,EAAE,aAAa,CAAC,EAAE,EAAE,MAAM,CAAC,EAAE,CAAC;AACrJ,CAAC;AAED,iFAAiF;AACjF,MAAM,UAAU,eAAe,CAAC,CAAmB,EAAE,IAAc,EAAE,GAAW;IAC9E,MAAM,KAAK,GAAG,OAAO,CAAC,IAAI,CAAC,CAAC;IAC5B,MAAM,MAAM,GAAG,KAAK,CAAC,GAAG,CAAC,QAAQ,CAAC,CAAC;IACnC,MAAM,EAAE,GAAG,KAAK,CAAC,SAAS,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,EAAE,KAAK,GAAG,IAAI,CAAC,CAAC,EAAE,KAAK,GAAG,CAAC,CAAC;IAChE,IAAI,EAAE,GAAG,CAAC;QAAE,MAAM,IAAI,KAAK,CAAC,yCAAyC,CAAC,CAAC;IACvE,OAAO,EAAE,CAAC,EAAE,CAAC,EAAE,IAAI,EAAE,qBAAqB,EAAE,MAAM,EAAE,CAAC,CAAC,MAAM,EAAE,GAAG,EAAE,KAAK,EAAE,EAAE,EAAE,IAAI,EAAE,MAAM,CAAC,MAAM,EAAE,IAAI,EAAE,aAAa,CAAC,EAAE,EAAE,MAAM,CAAC,EAAE,CAAC;AACvI,CAAC;AAED,6FAA6F;AAC7F,MAAM,UAAU,aAAa,CAAC,KAA+B,EAAE,CAAmB;IAChF,IAAI,CAAC;QACH,MAAM,CAAC,GAAG,KAAK,IAAK,EAAmB,CAAC;QACxC,IAAI,CAAC,CAAC,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC,IAAI,KAAK,kBAAkB,IAAI,CAAC,CAAC,MAAM,KAAK,CAAC,CAAC,MAAM;YAAE,OAAO,KAAK,CAAC;QACtF,MAAM,GAAG,GAAG,CAAC,CAAC,GAAG,IAAK,EAAW,CAAC;QAClC,IAAI,CAAC,CAAC,GAAG,CAAC,EAAE,GAAG,CAAC,CAAC,GAAG,IAAI,CAAC,CAAC,GAAG,GAAG,GAAG,CAAC,EAAE,CAAC;YAAE,OAAO,KAAK,CAAC,CAAC,sCAAsC;QAC7F,IAAI,GAAG,CAAC,EAAE,KAAK,GAAG,CAAC,EAAE;YAAE,OAAO,KAAK,CAAC,CAAC,6BAA6B;QAClE,MAAM,IAAI,GAAG,UAAU,CAAC,CAAC,CAAC,IAAI,CAAC,OAAO,CAAC,SAAS,EAAE,EAAE,CAAC,CAAC,CAAC;QACvD,OAAO,UAAU,CAAC,QAAQ,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC,KAAK,EAAE,CAAC,CAAC,IAAI,EAAE,CAAC,CAAC,IAAI,EAAE,IAAI,CAAC,CAAC;IAClE,CAAC;IAAC,MAAM,CAAC;QACP,OAAO,KAAK,CAAC,CAAC,kFAAkF;IAClG,CAAC;AACH,CAAC;AAED,oDAAoD;AACpD,MAAM,UAAU,gBAAgB,CAAC,KAAkC,EAAE,CAAmB;IACtF,IAAI,CAAC;QACH,MAAM,CAAC,GAAG,KAAK,IAAK,EAAsB,CAAC;QAC3C,IAAI,CAAC,CAAC,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC,IAAI,KAAK,qBAAqB,IAAI,CAAC,CAAC,MAAM,KAAK,CAAC,CAAC,MAAM;YAAE,OAAO,KAAK,CAAC;QACzF,MAAM,IAAI,GAAG,UAAU,CAAC,CAAC,CAAC,IAAI,CAAC,OAAO,CAAC,SAAS,EAAE,EAAE,CAAC,CAAC,CAAC;QACvD,OAAO,UAAU,CAAC,QAAQ,CAAC,EAAE,EAAE,EAAE,CAAC,CAAC,GAAG,EAAE,EAAE,EAAE,CAAC,CAAC,GAAG,EAAE,CAAC,EAAE,CAAC,CAAC,KAAK,EAAE,CAAC,CAAC,IAAI,EAAE,CAAC,CAAC,IAAI,EAAE,IAAI,CAAC,CAAC;IACvF,CAAC;IAAC,MAAM,CAAC;QACP,OAAO,KAAK,CAAC;IACf,CAAC;AACH,CAAC"}
|
|
1
|
+
{"version":3,"file":"index.js","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;GAuBG;AACH,OAAO,EAAE,UAAU,EAAE,UAAU,EAAE,MAAM,qBAAqB,CAAC;AAC7D,OAAO,EAAE,cAAc,EAAE,cAAc,EAAE,OAAO,EAAE,QAAQ,EAAE,UAAU,EAAE,eAAe,EAAE,MAAM,aAAa,CAAC;AAoB7G,yFAAyF;AACzF,gFAAgF;AAChF,SAAS,SAAS,CAAC,CAAS;IAC1B,IAAI,OAAO,CAAC,KAAK,QAAQ,IAAI,CAAC,CAAC,MAAM,KAAK,CAAC;QAAE,MAAM,IAAI,KAAK,CAAC,gCAAgC,CAAC,CAAC;IAC/F,KAAK,MAAM,CAAC,IAAI,CAAC,EAAE,CAAC;QAClB,IAAI,CAAC,CAAC,WAAW,CAAC,CAAC,CAAE,IAAI,MAAM;YAAE,MAAM,IAAI,KAAK,CAAC,4EAA4E,IAAI,CAAC,SAAS,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC;IACpJ,CAAC;AACH,CAAC;AAED,0FAA0F;AAC1F,SAAS,OAAO,CAAC,IAAc;IAC7B,KAAK,MAAM,CAAC,IAAI,IAAI;QAAE,SAAS,CAAC,CAAC,CAAC,CAAC;IACnC,MAAM,MAAM,GAAG,CAAC,GAAG,IAAI,GAAG,CAAC,IAAI,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC;IACzC,qFAAqF;IACrF,IAAI,MAAM,CAAC,MAAM,KAAK,CAAC;QAAE,OAAO,CAAC,EAAE,EAAE,EAAE,EAAE,EAAE,EAAE,EAAE,GAAG,EAAE,CAAC,CAAC;IACtD,MAAM,KAAK,GAAW,EAAE,CAAC;IACzB,wEAAwE;IACxE,KAAK,CAAC,IAAI,CAAC,EAAE,EAAE,EAAE,EAAE,EAAE,EAAE,EAAE,MAAM,CAAC,CAAC,CAAE,EAAE,CAAC,CAAC;IACvC,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,MAAM,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE,CAAC;QACvC,KAAK,CAAC,IAAI,CAAC,EAAE,EAAE,EAAE,MAAM,CAAC,CAAC,CAAE,EAAE,EAAE,EAAE,MAAM,CAAC,CAAC,CAAE,EAAE,CAAC,CAAC,CAAC,sCAAsC;QACtF,MAAM,IAAI,GAAG,CAAC,GAAG,CAAC,GAAG,MAAM,CAAC,MAAM,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,GAAG,CAAC,CAAE,CAAC,CAAC,CAAC,GAAG,CAAC;QAC1D,KAAK,CAAC,IAAI,CAAC,EAAE,EAAE,EAAE,MAAM,CAAC,CAAC,CAAE,EAAE,EAAE,EAAE,IAAI,EAAE,CAAC,CAAC,CAAC,6BAA6B;IACzE,CAAC;IACD,OAAO,KAAK,CAAC;AACf,CAAC;AAED,MAAM,QAAQ,GAAG,CAAC,CAAO,EAAc,EAAE,CAAC,QAAQ,CAAC,cAAc,CAAC,CAAuC,CAAC,CAAC,CAAC;AAE5G,+FAA+F;AAC/F,MAAM,UAAU,aAAa,CAAC,MAA4D;IACxF,IAAI,CAAC,MAAM,CAAC,KAAK;QAAE,MAAM,IAAI,KAAK,CAAC,mBAAmB,CAAC,CAAC;IACxD,IAAI,MAAM,CAAC,KAAK,CAAC,IAAI,CAAC,KAAK,CAAC,MAAM,CAAC,SAAS,CAAC,CAAC;QAAE,MAAM,IAAI,KAAK,CAAC,oCAAoC,CAAC,CAAC;IACtG,MAAM,KAAK,GAAG,OAAO,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC;IACnC,MAAM,IAAI,GAAG,UAAU,CAAC,UAAU,CAAC,KAAK,CAAC,GAAG,CAAC,QAAQ,CAAC,CAAC,CAAC,CAAC;IACzD,OAAO;QACL,CAAC,EAAE,CAAC;QACJ,IAAI,EAAE,mBAAmB;QACzB,MAAM,EAAE,MAAM,CAAC,KAAK;QACpB,IAAI,EAAE,IAAI,GAAG,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC,IAAI;QAC/B,IAAI,EAAE,UAAU,IAAI,EAAE;QACtB,UAAU,EAAE,MAAM,CAAC,SAAS;KAC7B,CAAC;AACJ,CAAC;AAED,MAAM,UAAU,YAAY,CAAC,CAAmB;IAC9C,OAAO,OAAO,CAAC,CAAuC,CAAC,CAAC;AAC1D,CAAC;AAED,0FAA0F;AAC1F,MAAM,UAAU,eAAe,CAAC,CAAmB;IACjD,OAAO,EAAE,MAAM,EAAE,cAAc,EAAE,OAAO,EAAE,EAAE,QAAQ,EAAE,CAAC,EAAE,aAAa,EAAE,YAAY,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC;AAC9F,CAAC;AAED,4EAA4E;AAC5E,gFAAgF;AAChF,MAAM,aAAa,GAAG,CAAC,KAAa,EAAE,MAAoB,EAAY,EAAE,CAAC,cAAc,CAAC,KAAK,EAAE,MAAM,CAAC,CAAC,GAAG,CAAC,UAAU,CAAC,CAAC;AACvH,MAAM,UAAU,GAAG,CAAC,IAAgB,EAAE,KAAa,EAAE,IAAY,EAAE,IAAc,EAAE,IAAgB,EAAW,EAAE,CAC9G,eAAe,CAAC,IAAI,EAAE,KAAK,EAAE,IAAI,EAAE,IAAI,CAAC,GAAG,CAAC,UAAU,CAAC,EAAE,IAAI,CAAC,CAAC;AAyBjE,yFAAyF;AACzF,MAAM,UAAU,YAAY,CAAC,CAAmB,EAAE,IAAc,EAAE,GAAW;IAC3E,SAAS,CAAC,GAAG,CAAC,CAAC;IACf,MAAM,MAAM,GAAG,CAAC,GAAG,IAAI,GAAG,CAAC,IAAI,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC;IACzC,IAAI,MAAM,CAAC,QAAQ,CAAC,GAAG,CAAC;QAAE,MAAM,IAAI,KAAK,CAAC,uCAAuC,CAAC,CAAC;IACnF,MAAM,KAAK,GAAG,OAAO,CAAC,IAAI,CAAC,CAAC;IAC5B,MAAM,MAAM,GAAG,KAAK,CAAC,GAAG,CAAC,QAAQ,CAAC,CAAC;IACnC,MAAM,EAAE,GAAG,KAAK,CAAC,SAAS,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,EAAE,GAAG,GAAG,IAAI,GAAG,GAAG,CAAC,CAAC,EAAE,IAAI,CAAC,CAAC,CAAC,CAAC,EAAE,KAAK,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC;IAChF,IAAI,EAAE,GAAG,CAAC;QAAE,MAAM,IAAI,KAAK,CAAC,sCAAsC,CAAC,CAAC;IACpE,OAAO,EAAE,CAAC,EAAE,CAAC,EAAE,IAAI,EAAE,kBAAkB,EAAE,MAAM,EAAE,CAAC,CAAC,MAAM,EAAE,GAAG,EAAE,GAAG,EAAE,KAAK,CAAC,EAAE,CAAE,EAAE,KAAK,EAAE,EAAE,EAAE,IAAI,EAAE,MAAM,CAAC,MAAM,EAAE,IAAI,EAAE,aAAa,CAAC,EAAE,EAAE,MAAM,CAAC,EAAE,CAAC;AACrJ,CAAC;AAED,iFAAiF;AACjF,MAAM,UAAU,eAAe,CAAC,CAAmB,EAAE,IAAc,EAAE,GAAW;IAC9E,MAAM,KAAK,GAAG,OAAO,CAAC,IAAI,CAAC,CAAC;IAC5B,MAAM,MAAM,GAAG,KAAK,CAAC,GAAG,CAAC,QAAQ,CAAC,CAAC;IACnC,MAAM,EAAE,GAAG,KAAK,CAAC,SAAS,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,EAAE,KAAK,GAAG,IAAI,CAAC,CAAC,EAAE,KAAK,GAAG,CAAC,CAAC;IAChE,IAAI,EAAE,GAAG,CAAC;QAAE,MAAM,IAAI,KAAK,CAAC,yCAAyC,CAAC,CAAC;IACvE,OAAO,EAAE,CAAC,EAAE,CAAC,EAAE,IAAI,EAAE,qBAAqB,EAAE,MAAM,EAAE,CAAC,CAAC,MAAM,EAAE,GAAG,EAAE,KAAK,EAAE,EAAE,EAAE,IAAI,EAAE,MAAM,CAAC,MAAM,EAAE,IAAI,EAAE,aAAa,CAAC,EAAE,EAAE,MAAM,CAAC,EAAE,CAAC;AACvI,CAAC;AAED,6FAA6F;AAC7F,MAAM,UAAU,aAAa,CAAC,KAA+B,EAAE,CAAmB;IAChF,IAAI,CAAC;QACH,MAAM,CAAC,GAAG,KAAK,IAAK,EAAmB,CAAC;QACxC,IAAI,CAAC,CAAC,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC,IAAI,KAAK,kBAAkB,IAAI,CAAC,CAAC,MAAM,KAAK,CAAC,CAAC,MAAM;YAAE,OAAO,KAAK,CAAC;QACtF,MAAM,GAAG,GAAG,CAAC,CAAC,GAAG,IAAK,EAAW,CAAC;QAClC,IAAI,CAAC,CAAC,GAAG,CAAC,EAAE,GAAG,CAAC,CAAC,GAAG,IAAI,CAAC,CAAC,GAAG,GAAG,GAAG,CAAC,EAAE,CAAC;YAAE,OAAO,KAAK,CAAC,CAAC,sCAAsC;QAC7F,IAAI,GAAG,CAAC,EAAE,KAAK,GAAG,CAAC,EAAE;YAAE,OAAO,KAAK,CAAC,CAAC,6BAA6B;QAClE,MAAM,IAAI,GAAG,UAAU,CAAC,CAAC,CAAC,IAAI,CAAC,OAAO,CAAC,SAAS,EAAE,EAAE,CAAC,CAAC,CAAC;QACvD,OAAO,UAAU,CAAC,QAAQ,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC,KAAK,EAAE,CAAC,CAAC,IAAI,EAAE,CAAC,CAAC,IAAI,EAAE,IAAI,CAAC,CAAC;IAClE,CAAC;IAAC,MAAM,CAAC;QACP,OAAO,KAAK,CAAC,CAAC,kFAAkF;IAClG,CAAC;AACH,CAAC;AAED,oDAAoD;AACpD,MAAM,UAAU,gBAAgB,CAAC,KAAkC,EAAE,CAAmB;IACtF,IAAI,CAAC;QACH,MAAM,CAAC,GAAG,KAAK,IAAK,EAAsB,CAAC;QAC3C,IAAI,CAAC,CAAC,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC,IAAI,KAAK,qBAAqB,IAAI,CAAC,CAAC,MAAM,KAAK,CAAC,CAAC,MAAM;YAAE,OAAO,KAAK,CAAC;QACzF,MAAM,IAAI,GAAG,UAAU,CAAC,CAAC,CAAC,IAAI,CAAC,OAAO,CAAC,SAAS,EAAE,EAAE,CAAC,CAAC,CAAC;QACvD,OAAO,UAAU,CAAC,QAAQ,CAAC,EAAE,EAAE,EAAE,CAAC,CAAC,GAAG,EAAE,EAAE,EAAE,CAAC,CAAC,GAAG,EAAE,CAAC,EAAE,CAAC,CAAC,KAAK,EAAE,CAAC,CAAC,IAAI,EAAE,CAAC,CAAC,IAAI,EAAE,IAAI,CAAC,CAAC;IACvF,CAAC;IAAC,MAAM,CAAC;QACP,OAAO,KAAK,CAAC;IACf,CAAC;AACH,CAAC;AACD,cAAc,iBAAiB,CAAC"}
|
|
@@ -0,0 +1,180 @@
|
|
|
1
|
+
import { type AbsenceProof, type MembershipProof, type TrainingManifest } from './index.js';
|
|
2
|
+
export interface DatasetCardFields {
|
|
3
|
+
name: string;
|
|
4
|
+
description: string;
|
|
5
|
+
sources: string[];
|
|
6
|
+
languages?: string[];
|
|
7
|
+
collectionPeriod?: {
|
|
8
|
+
from: string;
|
|
9
|
+
to: string;
|
|
10
|
+
};
|
|
11
|
+
preprocessing?: string[];
|
|
12
|
+
knownLimitations?: string[];
|
|
13
|
+
licence?: string;
|
|
14
|
+
}
|
|
15
|
+
/** A dataset card whose hash covers the manifest it describes: change either and the hash changes. */
|
|
16
|
+
export declare function datasetCard(m: TrainingManifest, fields: DatasetCardFields): {
|
|
17
|
+
hash: string;
|
|
18
|
+
name: string;
|
|
19
|
+
description: string;
|
|
20
|
+
sources: string[];
|
|
21
|
+
languages?: string[];
|
|
22
|
+
collectionPeriod?: {
|
|
23
|
+
from: string;
|
|
24
|
+
to: string;
|
|
25
|
+
};
|
|
26
|
+
preprocessing?: string[];
|
|
27
|
+
knownLimitations?: string[];
|
|
28
|
+
licence?: string;
|
|
29
|
+
v: number;
|
|
30
|
+
type: string;
|
|
31
|
+
manifest: string;
|
|
32
|
+
run_id: string;
|
|
33
|
+
size: number;
|
|
34
|
+
};
|
|
35
|
+
/**
|
|
36
|
+
* A public summary of training content for general-purpose AI models (AI Act Art. 53(1)(d)),
|
|
37
|
+
* laid out after the sections of the Commission's template: general information, data
|
|
38
|
+
* sources by kind, and data processing (opt-out handling, illegal-content removal).
|
|
39
|
+
*/
|
|
40
|
+
export declare function trainingContentSummary(m: TrainingManifest, input: {
|
|
41
|
+
provider: string;
|
|
42
|
+
model: string;
|
|
43
|
+
modalities: string[];
|
|
44
|
+
trainingDataSize: string;
|
|
45
|
+
sources: {
|
|
46
|
+
publicDatasets?: string[];
|
|
47
|
+
licensedPrivate?: string[];
|
|
48
|
+
webCrawled?: {
|
|
49
|
+
crawler: string;
|
|
50
|
+
period: string;
|
|
51
|
+
topDomains: string[];
|
|
52
|
+
};
|
|
53
|
+
userData?: string;
|
|
54
|
+
synthetic?: string;
|
|
55
|
+
other?: string;
|
|
56
|
+
};
|
|
57
|
+
processing: {
|
|
58
|
+
optOutMeasures: string;
|
|
59
|
+
illegalContentMeasures: string;
|
|
60
|
+
};
|
|
61
|
+
}): {
|
|
62
|
+
hash: string;
|
|
63
|
+
v: number;
|
|
64
|
+
type: string;
|
|
65
|
+
manifest: string;
|
|
66
|
+
general: {
|
|
67
|
+
provider: string;
|
|
68
|
+
model: string;
|
|
69
|
+
modalities: string[];
|
|
70
|
+
training_data_size: string;
|
|
71
|
+
};
|
|
72
|
+
sources: {
|
|
73
|
+
publicDatasets?: string[];
|
|
74
|
+
licensedPrivate?: string[];
|
|
75
|
+
webCrawled?: {
|
|
76
|
+
crawler: string;
|
|
77
|
+
period: string;
|
|
78
|
+
topDomains: string[];
|
|
79
|
+
};
|
|
80
|
+
userData?: string;
|
|
81
|
+
synthetic?: string;
|
|
82
|
+
other?: string;
|
|
83
|
+
};
|
|
84
|
+
processing: {
|
|
85
|
+
opt_out: string;
|
|
86
|
+
illegal_content: string;
|
|
87
|
+
};
|
|
88
|
+
};
|
|
89
|
+
/** For every opted-out work: an absence proof, or a violation when it is in the set. */
|
|
90
|
+
export declare function optOutsHonoured(m: TrainingManifest, refs: string[], optedOut: string[]): {
|
|
91
|
+
ok: boolean;
|
|
92
|
+
proofs: AbsenceProof[];
|
|
93
|
+
violations: string[];
|
|
94
|
+
};
|
|
95
|
+
/** The same check for an evaluation set: contamination is any eval item found in training. */
|
|
96
|
+
export declare function evalContamination(m: TrainingManifest, refs: string[], evalRefs: string[]): {
|
|
97
|
+
clean: boolean;
|
|
98
|
+
contaminated: string[];
|
|
99
|
+
rate: number;
|
|
100
|
+
absenceProofs: AbsenceProof[];
|
|
101
|
+
};
|
|
102
|
+
export interface SourceLicence {
|
|
103
|
+
source: string;
|
|
104
|
+
licence: string;
|
|
105
|
+
termsHash: string;
|
|
106
|
+
allowsTraining: boolean;
|
|
107
|
+
}
|
|
108
|
+
/** Commit per-source licence terms (as refs in a manifest) and list sources whose terms do not allow training. */
|
|
109
|
+
export declare function licenceLedger(runId: string, sources: SourceLicence[], createdAt: string): {
|
|
110
|
+
manifest: TrainingManifest;
|
|
111
|
+
refs: string[];
|
|
112
|
+
notAllowed: string[];
|
|
113
|
+
};
|
|
114
|
+
/** Prove one source's licence entry is in the committed licence ledger. */
|
|
115
|
+
export declare function licenceProof(ledger: ReturnType<typeof licenceLedger>, source: SourceLicence): MembershipProof;
|
|
116
|
+
/** Synthetic samples traced to the generator that made them and the samples it was seeded with. */
|
|
117
|
+
export interface SyntheticRecord {
|
|
118
|
+
ref: string;
|
|
119
|
+
generator: {
|
|
120
|
+
model: string;
|
|
121
|
+
version: string;
|
|
122
|
+
manifest?: string;
|
|
123
|
+
};
|
|
124
|
+
seeds: string[];
|
|
125
|
+
}
|
|
126
|
+
/** How many generations of synthetic data a sample sits on (0 for real data); deep chains signal model-collapse risk. */
|
|
127
|
+
export declare function syntheticDepth(records: SyntheticRecord[], ref: string, maxDepth?: number): number;
|
|
128
|
+
/** Bind a manifest to an attested training job (TEE or GPU attestation): one hash covering both. */
|
|
129
|
+
export declare function runAttestation(m: TrainingManifest, att: {
|
|
130
|
+
platform: string;
|
|
131
|
+
attestationHash: string;
|
|
132
|
+
imageHash: string;
|
|
133
|
+
startedAt: string;
|
|
134
|
+
endedAt: string;
|
|
135
|
+
}): {
|
|
136
|
+
hash: string;
|
|
137
|
+
platform: string;
|
|
138
|
+
attestationHash: string;
|
|
139
|
+
imageHash: string;
|
|
140
|
+
startedAt: string;
|
|
141
|
+
endedAt: string;
|
|
142
|
+
v: number;
|
|
143
|
+
type: string;
|
|
144
|
+
manifest: string;
|
|
145
|
+
};
|
|
146
|
+
export declare function verifyRunAttestation(b: ReturnType<typeof runAttestation>, m: TrainingManifest): boolean;
|
|
147
|
+
/** A fine-tune's own manifest, linked to its base model's manifest. */
|
|
148
|
+
export declare function fineTuneManifest(base: TrainingManifest, input: {
|
|
149
|
+
runId: string;
|
|
150
|
+
refs: string[];
|
|
151
|
+
createdAt: string;
|
|
152
|
+
}): {
|
|
153
|
+
manifest: TrainingManifest;
|
|
154
|
+
base: string;
|
|
155
|
+
base_run: string;
|
|
156
|
+
};
|
|
157
|
+
/** Split a pool (minor units) among contributors by how many of their works were included. Exact; remainders by largest fraction. */
|
|
158
|
+
export declare function royaltyByInclusion(included: string[], contributorOf: Record<string, string>, poolMinor: bigint): {
|
|
159
|
+
payouts: Array<{
|
|
160
|
+
contributor: string;
|
|
161
|
+
works: number;
|
|
162
|
+
minor: string;
|
|
163
|
+
}>;
|
|
164
|
+
};
|
|
165
|
+
/** Which runs included a suspect sample: a membership proof for each, an absence proof for the rest. */
|
|
166
|
+
export declare function poisoningForensics(sampleRef: string, runs: Array<{
|
|
167
|
+
manifest: TrainingManifest;
|
|
168
|
+
refs: string[];
|
|
169
|
+
}>): {
|
|
170
|
+
sample: string;
|
|
171
|
+
affected: {
|
|
172
|
+
run: string;
|
|
173
|
+
proof: MembershipProof;
|
|
174
|
+
}[];
|
|
175
|
+
clear: {
|
|
176
|
+
run: string;
|
|
177
|
+
proof: AbsenceProof;
|
|
178
|
+
}[];
|
|
179
|
+
};
|
|
180
|
+
//# sourceMappingURL=provenance.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"provenance.d.ts","sourceRoot":"","sources":["../src/provenance.ts"],"names":[],"mappings":"AAOA,OAAO,EAA8D,KAAK,YAAY,EAAE,KAAK,eAAe,EAAE,KAAK,gBAAgB,EAAE,MAAM,YAAY,CAAC;AAMxJ,MAAM,WAAW,iBAAiB;IAChC,IAAI,EAAE,MAAM,CAAC;IACb,WAAW,EAAE,MAAM,CAAC;IACpB,OAAO,EAAE,MAAM,EAAE,CAAC;IAClB,SAAS,CAAC,EAAE,MAAM,EAAE,CAAC;IACrB,gBAAgB,CAAC,EAAE;QAAE,IAAI,EAAE,MAAM,CAAC;QAAC,EAAE,EAAE,MAAM,CAAA;KAAE,CAAC;IAChD,aAAa,CAAC,EAAE,MAAM,EAAE,CAAC;IACzB,gBAAgB,CAAC,EAAE,MAAM,EAAE,CAAC;IAC5B,OAAO,CAAC,EAAE,MAAM,CAAC;CAClB;AAED,sGAAsG;AACtG,wBAAgB,WAAW,CAAC,CAAC,EAAE,gBAAgB,EAAE,MAAM,EAAE,iBAAiB;;UAXlE,MAAM;iBACC,MAAM;aACV,MAAM,EAAE;gBACL,MAAM,EAAE;uBACD;QAAE,IAAI,EAAE,MAAM,CAAC;QAAC,EAAE,EAAE,MAAM,CAAA;KAAE;oBAC/B,MAAM,EAAE;uBACL,MAAM,EAAE;cACjB,MAAM;;;;;;EAOjB;AAED;;;;GAIG;AACH,wBAAgB,sBAAsB,CAAC,CAAC,EAAE,gBAAgB,EAAE,KAAK,EAAE;IACjE,QAAQ,EAAE,MAAM,CAAC;IAAC,KAAK,EAAE,MAAM,CAAC;IAAC,UAAU,EAAE,MAAM,EAAE,CAAC;IAAC,gBAAgB,EAAE,MAAM,CAAC;IAChF,OAAO,EAAE;QAAE,cAAc,CAAC,EAAE,MAAM,EAAE,CAAC;QAAC,eAAe,CAAC,EAAE,MAAM,EAAE,CAAC;QAAC,UAAU,CAAC,EAAE;YAAE,OAAO,EAAE,MAAM,CAAC;YAAC,MAAM,EAAE,MAAM,CAAC;YAAC,UAAU,EAAE,MAAM,EAAE,CAAA;SAAE,CAAC;QAAC,QAAQ,CAAC,EAAE,MAAM,CAAC;QAAC,SAAS,CAAC,EAAE,MAAM,CAAC;QAAC,KAAK,CAAC,EAAE,MAAM,CAAA;KAAE,CAAC;IAClM,UAAU,EAAE;QAAE,cAAc,EAAE,MAAM,CAAC;QAAC,sBAAsB,EAAE,MAAM,CAAA;KAAE,CAAC;CACxE;;;;;;;;;;;;yBAF6B,MAAM,EAAE;0BAAoB,MAAM,EAAE;qBAAe;YAAE,OAAO,EAAE,MAAM,CAAC;YAAC,MAAM,EAAE,MAAM,CAAC;YAAC,UAAU,EAAE,MAAM,EAAE,CAAA;SAAE;mBAAa,MAAM;oBAAc,MAAM;gBAAU,MAAM;;;;;;EAUhM;AAMD,wFAAwF;AACxF,wBAAgB,eAAe,CAAC,CAAC,EAAE,gBAAgB,EAAE,IAAI,EAAE,MAAM,EAAE,EAAE,QAAQ,EAAE,MAAM,EAAE;;;;EAMtF;AAED,8FAA8F;AAC9F,wBAAgB,iBAAiB,CAAC,CAAC,EAAE,gBAAgB,EAAE,IAAI,EAAE,MAAM,EAAE,EAAE,QAAQ,EAAE,MAAM,EAAE;;;;;EAGxF;AAMD,MAAM,WAAW,aAAa;IAC5B,MAAM,EAAE,MAAM,CAAC;IACf,OAAO,EAAE,MAAM,CAAC;IAChB,SAAS,EAAE,MAAM,CAAC;IAClB,cAAc,EAAE,OAAO,CAAC;CACzB;AAED,kHAAkH;AAClH,wBAAgB,aAAa,CAAC,KAAK,EAAE,MAAM,EAAE,OAAO,EAAE,aAAa,EAAE,EAAE,SAAS,EAAE,MAAM;;;;EAGvF;AAED,2EAA2E;AAC3E,wBAAgB,YAAY,CAAC,MAAM,EAAE,UAAU,CAAC,OAAO,aAAa,CAAC,EAAE,MAAM,EAAE,aAAa,GAAG,eAAe,CAE7G;AAED,mGAAmG;AACnG,MAAM,WAAW,eAAe;IAC9B,GAAG,EAAE,MAAM,CAAC;IACZ,SAAS,EAAE;QAAE,KAAK,EAAE,MAAM,CAAC;QAAC,OAAO,EAAE,MAAM,CAAC;QAAC,QAAQ,CAAC,EAAE,MAAM,CAAA;KAAE,CAAC;IACjE,KAAK,EAAE,MAAM,EAAE,CAAC;CACjB;AAED,yHAAyH;AACzH,wBAAgB,cAAc,CAAC,OAAO,EAAE,eAAe,EAAE,EAAE,GAAG,EAAE,MAAM,EAAE,QAAQ,SAAK,GAAG,MAAM,CAQ7F;AAED,oGAAoG;AACpG,wBAAgB,cAAc,CAAC,CAAC,EAAE,gBAAgB,EAAE,GAAG,EAAE;IAAE,QAAQ,EAAE,MAAM,CAAC;IAAC,eAAe,EAAE,MAAM,CAAC;IAAC,SAAS,EAAE,MAAM,CAAC;IAAC,SAAS,EAAE,MAAM,CAAC;IAAC,OAAO,EAAE,MAAM,CAAA;CAAE;;cAAxF,MAAM;qBAAmB,MAAM;eAAa,MAAM;eAAa,MAAM;aAAW,MAAM;;;;EAG1J;AAED,wBAAgB,oBAAoB,CAAC,CAAC,EAAE,UAAU,CAAC,OAAO,cAAc,CAAC,EAAE,CAAC,EAAE,gBAAgB,GAAG,OAAO,CAGvG;AAED,uEAAuE;AACvE,wBAAgB,gBAAgB,CAAC,IAAI,EAAE,gBAAgB,EAAE,KAAK,EAAE;IAAE,KAAK,EAAE,MAAM,CAAC;IAAC,IAAI,EAAE,MAAM,EAAE,CAAC;IAAC,SAAS,EAAE,MAAM,CAAA;CAAE;;;;EAEnH;AAMD,qIAAqI;AACrI,wBAAgB,kBAAkB,CAAC,QAAQ,EAAE,MAAM,EAAE,EAAE,aAAa,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,EAAE,SAAS,EAAE,MAAM;aAOnE,KAAK,CAAC;QAAE,WAAW,EAAE,MAAM,CAAC;QAAC,KAAK,EAAE,MAAM,CAAC;QAAC,KAAK,EAAE,MAAM,CAAA;KAAE,CAAC;EASvG;AAED,wGAAwG;AACxG,wBAAgB,kBAAkB,CAAC,SAAS,EAAE,MAAM,EAAE,IAAI,EAAE,KAAK,CAAC;IAAE,QAAQ,EAAE,gBAAgB,CAAC;IAAC,IAAI,EAAE,MAAM,EAAE,CAAA;CAAE,CAAC;;;aAClF,MAAM;eAAS,eAAe;;;aACjC,MAAM;eAAS,YAAY;;EAMtD"}
|
|
@@ -0,0 +1,114 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Training-data provenance, further: dataset cards and a training-content summary bound to
|
|
3
|
+
* the manifest, opt-out and contamination checks with absence proofs, a licence ledger,
|
|
4
|
+
* synthetic-data lineage, run attestations, fine-tune deltas, royalty accounting by
|
|
5
|
+
* inclusion and poisoning forensics across runs.
|
|
6
|
+
*/
|
|
7
|
+
import { jcsHash } from '@zanii/core';
|
|
8
|
+
import { buildManifest, manifestHash, proveAbsence, proveMembership } from './index.js';
|
|
9
|
+
/** A dataset card whose hash covers the manifest it describes: change either and the hash changes. */
|
|
10
|
+
export function datasetCard(m, fields) {
|
|
11
|
+
const card = { v: 1, type: 'training.dataset_card', manifest: manifestHash(m), run_id: m.run_id, size: m.size, ...fields };
|
|
12
|
+
return { ...card, hash: jcsHash(card) };
|
|
13
|
+
}
|
|
14
|
+
/**
|
|
15
|
+
* A public summary of training content for general-purpose AI models (AI Act Art. 53(1)(d)),
|
|
16
|
+
* laid out after the sections of the Commission's template: general information, data
|
|
17
|
+
* sources by kind, and data processing (opt-out handling, illegal-content removal).
|
|
18
|
+
*/
|
|
19
|
+
export function trainingContentSummary(m, input) {
|
|
20
|
+
const summary = {
|
|
21
|
+
v: 1, type: 'training.content_summary', manifest: manifestHash(m),
|
|
22
|
+
general: { provider: input.provider, model: input.model, modalities: input.modalities, training_data_size: input.trainingDataSize },
|
|
23
|
+
sources: input.sources,
|
|
24
|
+
processing: { opt_out: input.processing.optOutMeasures, illegal_content: input.processing.illegalContentMeasures },
|
|
25
|
+
};
|
|
26
|
+
return { ...summary, hash: jcsHash(summary) };
|
|
27
|
+
}
|
|
28
|
+
// ------------------------------------------------------------------------------ //
|
|
29
|
+
// Exclusions: opt-outs and evaluation sets
|
|
30
|
+
// ------------------------------------------------------------------------------ //
|
|
31
|
+
/** For every opted-out work: an absence proof, or a violation when it is in the set. */
|
|
32
|
+
export function optOutsHonoured(m, refs, optedOut) {
|
|
33
|
+
const present = new Set(refs);
|
|
34
|
+
const proofs = [];
|
|
35
|
+
const violations = [];
|
|
36
|
+
for (const r of optedOut)
|
|
37
|
+
present.has(r) ? violations.push(r) : proofs.push(proveAbsence(m, refs, r));
|
|
38
|
+
return { ok: violations.length === 0, proofs, violations };
|
|
39
|
+
}
|
|
40
|
+
/** The same check for an evaluation set: contamination is any eval item found in training. */
|
|
41
|
+
export function evalContamination(m, refs, evalRefs) {
|
|
42
|
+
const r = optOutsHonoured(m, refs, evalRefs);
|
|
43
|
+
return { clean: r.ok, contaminated: r.violations, rate: evalRefs.length ? r.violations.length / evalRefs.length : 0, absenceProofs: r.proofs };
|
|
44
|
+
}
|
|
45
|
+
/** Commit per-source licence terms (as refs in a manifest) and list sources whose terms do not allow training. */
|
|
46
|
+
export function licenceLedger(runId, sources, createdAt) {
|
|
47
|
+
const refs = sources.map((s) => jcsHash(s));
|
|
48
|
+
return { manifest: buildManifest({ runId: `${runId}#licences`, refs, createdAt }), refs, notAllowed: sources.filter((s) => !s.allowsTraining).map((s) => s.source) };
|
|
49
|
+
}
|
|
50
|
+
/** Prove one source's licence entry is in the committed licence ledger. */
|
|
51
|
+
export function licenceProof(ledger, source) {
|
|
52
|
+
return proveMembership(ledger.manifest, ledger.refs, jcsHash(source));
|
|
53
|
+
}
|
|
54
|
+
/** How many generations of synthetic data a sample sits on (0 for real data); deep chains signal model-collapse risk. */
|
|
55
|
+
export function syntheticDepth(records, ref, maxDepth = 32) {
|
|
56
|
+
const by = new Map(records.map((r) => [r.ref, r]));
|
|
57
|
+
const depth = (x, d) => {
|
|
58
|
+
const r = by.get(x);
|
|
59
|
+
if (!r || d >= maxDepth)
|
|
60
|
+
return 0;
|
|
61
|
+
return 1 + Math.max(0, ...r.seeds.map((s) => depth(s, d + 1)));
|
|
62
|
+
};
|
|
63
|
+
return depth(ref, 0);
|
|
64
|
+
}
|
|
65
|
+
/** Bind a manifest to an attested training job (TEE or GPU attestation): one hash covering both. */
|
|
66
|
+
export function runAttestation(m, att) {
|
|
67
|
+
const binding = { v: 1, type: 'training.run_attestation', manifest: manifestHash(m), ...att };
|
|
68
|
+
return { ...binding, hash: jcsHash(binding) };
|
|
69
|
+
}
|
|
70
|
+
export function verifyRunAttestation(b, m) {
|
|
71
|
+
const { hash, ...rest } = b;
|
|
72
|
+
return rest.manifest === manifestHash(m) && jcsHash(rest) === hash;
|
|
73
|
+
}
|
|
74
|
+
/** A fine-tune's own manifest, linked to its base model's manifest. */
|
|
75
|
+
export function fineTuneManifest(base, input) {
|
|
76
|
+
return { manifest: buildManifest(input), base: manifestHash(base), base_run: base.run_id };
|
|
77
|
+
}
|
|
78
|
+
// ------------------------------------------------------------------------------ //
|
|
79
|
+
// Royalties and forensics
|
|
80
|
+
// ------------------------------------------------------------------------------ //
|
|
81
|
+
/** Split a pool (minor units) among contributors by how many of their works were included. Exact; remainders by largest fraction. */
|
|
82
|
+
export function royaltyByInclusion(included, contributorOf, poolMinor) {
|
|
83
|
+
const counts = new Map();
|
|
84
|
+
for (const r of new Set(included)) {
|
|
85
|
+
const c = contributorOf[r];
|
|
86
|
+
if (c)
|
|
87
|
+
counts.set(c, (counts.get(c) ?? 0n) + 1n);
|
|
88
|
+
}
|
|
89
|
+
const total = [...counts.values()].reduce((a, b) => a + b, 0n);
|
|
90
|
+
if (total === 0n)
|
|
91
|
+
return { payouts: [] };
|
|
92
|
+
const rows = [...counts].map(([contributor, n]) => ({ contributor, n, q: (poolMinor * n) / total, r: (poolMinor * n) % total }));
|
|
93
|
+
let left = poolMinor - rows.reduce((s, x) => s + x.q, 0n);
|
|
94
|
+
for (const x of [...rows].sort((a, b) => (b.r > a.r ? 1 : b.r < a.r ? -1 : a.contributor.localeCompare(b.contributor)))) {
|
|
95
|
+
if (left === 0n)
|
|
96
|
+
break;
|
|
97
|
+
x.q += 1n;
|
|
98
|
+
left -= 1n;
|
|
99
|
+
}
|
|
100
|
+
return { payouts: rows.map((x) => ({ contributor: x.contributor, works: Number(x.n), minor: x.q.toString() })) };
|
|
101
|
+
}
|
|
102
|
+
/** Which runs included a suspect sample: a membership proof for each, an absence proof for the rest. */
|
|
103
|
+
export function poisoningForensics(sampleRef, runs) {
|
|
104
|
+
const affected = [];
|
|
105
|
+
const clear = [];
|
|
106
|
+
for (const r of runs) {
|
|
107
|
+
if (r.refs.includes(sampleRef))
|
|
108
|
+
affected.push({ run: r.manifest.run_id, proof: proveMembership(r.manifest, r.refs, sampleRef) });
|
|
109
|
+
else
|
|
110
|
+
clear.push({ run: r.manifest.run_id, proof: proveAbsence(r.manifest, r.refs, sampleRef) });
|
|
111
|
+
}
|
|
112
|
+
return { sample: sampleRef, affected, clear };
|
|
113
|
+
}
|
|
114
|
+
//# sourceMappingURL=provenance.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"provenance.js","sourceRoot":"","sources":["../src/provenance.ts"],"names":[],"mappings":"AAAA;;;;;GAKG;AACH,OAAO,EAAE,OAAO,EAAE,MAAM,aAAa,CAAC;AACtC,OAAO,EAAE,aAAa,EAAE,YAAY,EAAE,YAAY,EAAE,eAAe,EAAkE,MAAM,YAAY,CAAC;AAiBxJ,sGAAsG;AACtG,MAAM,UAAU,WAAW,CAAC,CAAmB,EAAE,MAAyB;IACxE,MAAM,IAAI,GAAG,EAAE,CAAC,EAAE,CAAC,EAAE,IAAI,EAAE,uBAAuB,EAAE,QAAQ,EAAE,YAAY,CAAC,CAAC,CAAC,EAAE,MAAM,EAAE,CAAC,CAAC,MAAM,EAAE,IAAI,EAAE,CAAC,CAAC,IAAI,EAAE,GAAG,MAAM,EAAE,CAAC;IAC3H,OAAO,EAAE,GAAG,IAAI,EAAE,IAAI,EAAE,OAAO,CAAC,IAAI,CAAC,EAAE,CAAC;AAC1C,CAAC;AAED;;;;GAIG;AACH,MAAM,UAAU,sBAAsB,CAAC,CAAmB,EAAE,KAI3D;IACC,MAAM,OAAO,GAAG;QACd,CAAC,EAAE,CAAC,EAAE,IAAI,EAAE,0BAA0B,EAAE,QAAQ,EAAE,YAAY,CAAC,CAAC,CAAC;QACjE,OAAO,EAAE,EAAE,QAAQ,EAAE,KAAK,CAAC,QAAQ,EAAE,KAAK,EAAE,KAAK,CAAC,KAAK,EAAE,UAAU,EAAE,KAAK,CAAC,UAAU,EAAE,kBAAkB,EAAE,KAAK,CAAC,gBAAgB,EAAE;QACnI,OAAO,EAAE,KAAK,CAAC,OAAO;QACtB,UAAU,EAAE,EAAE,OAAO,EAAE,KAAK,CAAC,UAAU,CAAC,cAAc,EAAE,eAAe,EAAE,KAAK,CAAC,UAAU,CAAC,sBAAsB,EAAE;KACnH,CAAC;IACF,OAAO,EAAE,GAAG,OAAO,EAAE,IAAI,EAAE,OAAO,CAAC,OAAO,CAAC,EAAE,CAAC;AAChD,CAAC;AAED,oFAAoF;AACpF,2CAA2C;AAC3C,oFAAoF;AAEpF,wFAAwF;AACxF,MAAM,UAAU,eAAe,CAAC,CAAmB,EAAE,IAAc,EAAE,QAAkB;IACrF,MAAM,OAAO,GAAG,IAAI,GAAG,CAAC,IAAI,CAAC,CAAC;IAC9B,MAAM,MAAM,GAAmB,EAAE,CAAC;IAClC,MAAM,UAAU,GAAa,EAAE,CAAC;IAChC,KAAK,MAAM,CAAC,IAAI,QAAQ;QAAE,OAAO,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,UAAU,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC,IAAI,CAAC,YAAY,CAAC,CAAC,EAAE,IAAI,EAAE,CAAC,CAAC,CAAC,CAAC;IACtG,OAAO,EAAE,EAAE,EAAE,UAAU,CAAC,MAAM,KAAK,CAAC,EAAE,MAAM,EAAE,UAAU,EAAE,CAAC;AAC7D,CAAC;AAED,8FAA8F;AAC9F,MAAM,UAAU,iBAAiB,CAAC,CAAmB,EAAE,IAAc,EAAE,QAAkB;IACvF,MAAM,CAAC,GAAG,eAAe,CAAC,CAAC,EAAE,IAAI,EAAE,QAAQ,CAAC,CAAC;IAC7C,OAAO,EAAE,KAAK,EAAE,CAAC,CAAC,EAAE,EAAE,YAAY,EAAE,CAAC,CAAC,UAAU,EAAE,IAAI,EAAE,QAAQ,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,CAAC,UAAU,CAAC,MAAM,GAAG,QAAQ,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,EAAE,aAAa,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC;AACjJ,CAAC;AAaD,kHAAkH;AAClH,MAAM,UAAU,aAAa,CAAC,KAAa,EAAE,OAAwB,EAAE,SAAiB;IACtF,MAAM,IAAI,GAAG,OAAO,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,OAAO,CAAC,CAAU,CAAC,CAAC,CAAC;IACrD,OAAO,EAAE,QAAQ,EAAE,aAAa,CAAC,EAAE,KAAK,EAAE,GAAG,KAAK,WAAW,EAAE,IAAI,EAAE,SAAS,EAAE,CAAC,EAAE,IAAI,EAAE,UAAU,EAAE,OAAO,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,CAAC,cAAc,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,MAAM,CAAC,EAAE,CAAC;AACvK,CAAC;AAED,2EAA2E;AAC3E,MAAM,UAAU,YAAY,CAAC,MAAwC,EAAE,MAAqB;IAC1F,OAAO,eAAe,CAAC,MAAM,CAAC,QAAQ,EAAE,MAAM,CAAC,IAAI,EAAE,OAAO,CAAC,MAAe,CAAC,CAAC,CAAC;AACjF,CAAC;AASD,yHAAyH;AACzH,MAAM,UAAU,cAAc,CAAC,OAA0B,EAAE,GAAW,EAAE,QAAQ,GAAG,EAAE;IACnF,MAAM,EAAE,GAAG,IAAI,GAAG,CAAC,OAAO,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,CAAC,GAAG,EAAE,CAAC,CAAC,CAAC,CAAC,CAAC;IACnD,MAAM,KAAK,GAAG,CAAC,CAAS,EAAE,CAAS,EAAU,EAAE;QAC7C,MAAM,CAAC,GAAG,EAAE,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC;QACpB,IAAI,CAAC,CAAC,IAAI,CAAC,IAAI,QAAQ;YAAE,OAAO,CAAC,CAAC;QAClC,OAAO,CAAC,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,GAAG,CAAC,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC;IACjE,CAAC,CAAC;IACF,OAAO,KAAK,CAAC,GAAG,EAAE,CAAC,CAAC,CAAC;AACvB,CAAC;AAED,oGAAoG;AACpG,MAAM,UAAU,cAAc,CAAC,CAAmB,EAAE,GAAyG;IAC3J,MAAM,OAAO,GAAG,EAAE,CAAC,EAAE,CAAC,EAAE,IAAI,EAAE,0BAA0B,EAAE,QAAQ,EAAE,YAAY,CAAC,CAAC,CAAC,EAAE,GAAG,GAAG,EAAE,CAAC;IAC9F,OAAO,EAAE,GAAG,OAAO,EAAE,IAAI,EAAE,OAAO,CAAC,OAAO,CAAC,EAAE,CAAC;AAChD,CAAC;AAED,MAAM,UAAU,oBAAoB,CAAC,CAAoC,EAAE,CAAmB;IAC5F,MAAM,EAAE,IAAI,EAAE,GAAG,IAAI,EAAE,GAAG,CAAC,CAAC;IAC5B,OAAO,IAAI,CAAC,QAAQ,KAAK,YAAY,CAAC,CAAC,CAAC,IAAI,OAAO,CAAC,IAAI,CAAC,KAAK,IAAI,CAAC;AACrE,CAAC;AAED,uEAAuE;AACvE,MAAM,UAAU,gBAAgB,CAAC,IAAsB,EAAE,KAA2D;IAClH,OAAO,EAAE,QAAQ,EAAE,aAAa,CAAC,KAAK,CAAC,EAAE,IAAI,EAAE,YAAY,CAAC,IAAI,CAAC,EAAE,QAAQ,EAAE,IAAI,CAAC,MAAM,EAAE,CAAC;AAC7F,CAAC;AAED,oFAAoF;AACpF,0BAA0B;AAC1B,oFAAoF;AAEpF,qIAAqI;AACrI,MAAM,UAAU,kBAAkB,CAAC,QAAkB,EAAE,aAAqC,EAAE,SAAiB;IAC7G,MAAM,MAAM,GAAG,IAAI,GAAG,EAAkB,CAAC;IACzC,KAAK,MAAM,CAAC,IAAI,IAAI,GAAG,CAAC,QAAQ,CAAC,EAAE,CAAC;QAClC,MAAM,CAAC,GAAG,aAAa,CAAC,CAAC,CAAC,CAAC;QAC3B,IAAI,CAAC;YAAE,MAAM,CAAC,GAAG,CAAC,CAAC,EAAE,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC,GAAG,EAAE,CAAC,CAAC;IACnD,CAAC;IACD,MAAM,KAAK,GAAG,CAAC,GAAG,MAAM,CAAC,MAAM,EAAE,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,GAAG,CAAC,EAAE,EAAE,CAAC,CAAC;IAC/D,IAAI,KAAK,KAAK,EAAE;QAAE,OAAO,EAAE,OAAO,EAAE,EAAkE,EAAE,CAAC;IACzG,MAAM,IAAI,GAAG,CAAC,GAAG,MAAM,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,WAAW,EAAE,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,EAAE,WAAW,EAAE,CAAC,EAAE,CAAC,EAAE,CAAC,SAAS,GAAG,CAAC,CAAC,GAAG,KAAK,EAAE,CAAC,EAAE,CAAC,SAAS,GAAG,CAAC,CAAC,GAAG,KAAK,EAAE,CAAC,CAAC,CAAC;IACjI,IAAI,IAAI,GAAG,SAAS,GAAG,IAAI,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC;IAC1D,KAAK,MAAM,CAAC,IAAI,CAAC,GAAG,IAAI,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,WAAW,CAAC,aAAa,CAAC,CAAC,CAAC,WAAW,CAAC,CAAC,CAAC,EAAE,CAAC;QACxH,IAAI,IAAI,KAAK,EAAE;YAAE,MAAM;QACvB,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC;QACV,IAAI,IAAI,EAAE,CAAC;IACb,CAAC;IACD,OAAO,EAAE,OAAO,EAAE,IAAI,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,EAAE,WAAW,EAAE,CAAC,CAAC,WAAW,EAAE,KAAK,EAAE,MAAM,CAAC,CAAC,CAAC,CAAC,CAAC,EAAE,KAAK,EAAE,CAAC,CAAC,CAAC,CAAC,QAAQ,EAAE,EAAE,CAAC,CAAC,EAAE,CAAC;AACnH,CAAC;AAED,wGAAwG;AACxG,MAAM,UAAU,kBAAkB,CAAC,SAAiB,EAAE,IAA2D;IAC/G,MAAM,QAAQ,GAAmD,EAAE,CAAC;IACpE,MAAM,KAAK,GAAgD,EAAE,CAAC;IAC9D,KAAK,MAAM,CAAC,IAAI,IAAI,EAAE,CAAC;QACrB,IAAI,CAAC,CAAC,IAAI,CAAC,QAAQ,CAAC,SAAS,CAAC;YAAE,QAAQ,CAAC,IAAI,CAAC,EAAE,GAAG,EAAE,CAAC,CAAC,QAAQ,CAAC,MAAM,EAAE,KAAK,EAAE,eAAe,CAAC,CAAC,CAAC,QAAQ,EAAE,CAAC,CAAC,IAAI,EAAE,SAAS,CAAC,EAAE,CAAC,CAAC;;YAC5H,KAAK,CAAC,IAAI,CAAC,EAAE,GAAG,EAAE,CAAC,CAAC,QAAQ,CAAC,MAAM,EAAE,KAAK,EAAE,YAAY,CAAC,CAAC,CAAC,QAAQ,EAAE,CAAC,CAAC,IAAI,EAAE,SAAS,CAAC,EAAE,CAAC,CAAC;IAClG,CAAC;IACD,OAAO,EAAE,MAAM,EAAE,SAAS,EAAE,QAAQ,EAAE,KAAK,EAAE,CAAC;AAChD,CAAC"}
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@zanii/train",
|
|
3
|
-
"version": "0.2.
|
|
3
|
+
"version": "0.2.2",
|
|
4
4
|
"description": "Training-data provenance: hash a training set into a sorted-Merkle manifest, then PROVE a named subject's data was absent from a real training run - a non-membership proof anyone can check offline. 'It was never used to train a model' becomes a yes, not a promise. Proves what was recorded went in; cannot prove an off-manifest copy didn't (state the enclave boundary).",
|
|
5
5
|
"license": "Apache-2.0",
|
|
6
6
|
"homepage": "https://ledger.zanii.agency",
|
|
@@ -32,7 +32,7 @@
|
|
|
32
32
|
},
|
|
33
33
|
"dependencies": {
|
|
34
34
|
"@noble/hashes": "^1.7.1",
|
|
35
|
-
"@zanii/core": "^0.5.
|
|
35
|
+
"@zanii/core": "^0.5.4"
|
|
36
36
|
},
|
|
37
37
|
"scripts": {
|
|
38
38
|
"build": "tsc -p tsconfig.build.json",
|
package/src/index.ts
CHANGED
|
@@ -0,0 +1,157 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Training-data provenance, further: dataset cards and a training-content summary bound to
|
|
3
|
+
* the manifest, opt-out and contamination checks with absence proofs, a licence ledger,
|
|
4
|
+
* synthetic-data lineage, run attestations, fine-tune deltas, royalty accounting by
|
|
5
|
+
* inclusion and poisoning forensics across runs.
|
|
6
|
+
*/
|
|
7
|
+
import { jcsHash } from '@zanii/core';
|
|
8
|
+
import { buildManifest, manifestHash, proveAbsence, proveMembership, type AbsenceProof, type MembershipProof, type TrainingManifest } from './index.js';
|
|
9
|
+
|
|
10
|
+
// ------------------------------------------------------------------------------ //
|
|
11
|
+
// Documentation bound to the manifest
|
|
12
|
+
// ------------------------------------------------------------------------------ //
|
|
13
|
+
|
|
14
|
+
export interface DatasetCardFields {
|
|
15
|
+
name: string;
|
|
16
|
+
description: string;
|
|
17
|
+
sources: string[];
|
|
18
|
+
languages?: string[];
|
|
19
|
+
collectionPeriod?: { from: string; to: string };
|
|
20
|
+
preprocessing?: string[];
|
|
21
|
+
knownLimitations?: string[];
|
|
22
|
+
licence?: string;
|
|
23
|
+
}
|
|
24
|
+
|
|
25
|
+
/** A dataset card whose hash covers the manifest it describes: change either and the hash changes. */
|
|
26
|
+
export function datasetCard(m: TrainingManifest, fields: DatasetCardFields) {
|
|
27
|
+
const card = { v: 1, type: 'training.dataset_card', manifest: manifestHash(m), run_id: m.run_id, size: m.size, ...fields };
|
|
28
|
+
return { ...card, hash: jcsHash(card) };
|
|
29
|
+
}
|
|
30
|
+
|
|
31
|
+
/**
|
|
32
|
+
* A public summary of training content for general-purpose AI models (AI Act Art. 53(1)(d)),
|
|
33
|
+
* laid out after the sections of the Commission's template: general information, data
|
|
34
|
+
* sources by kind, and data processing (opt-out handling, illegal-content removal).
|
|
35
|
+
*/
|
|
36
|
+
export function trainingContentSummary(m: TrainingManifest, input: {
|
|
37
|
+
provider: string; model: string; modalities: string[]; trainingDataSize: string;
|
|
38
|
+
sources: { publicDatasets?: string[]; licensedPrivate?: string[]; webCrawled?: { crawler: string; period: string; topDomains: string[] }; userData?: string; synthetic?: string; other?: string };
|
|
39
|
+
processing: { optOutMeasures: string; illegalContentMeasures: string };
|
|
40
|
+
}) {
|
|
41
|
+
const summary = {
|
|
42
|
+
v: 1, type: 'training.content_summary', manifest: manifestHash(m),
|
|
43
|
+
general: { provider: input.provider, model: input.model, modalities: input.modalities, training_data_size: input.trainingDataSize },
|
|
44
|
+
sources: input.sources,
|
|
45
|
+
processing: { opt_out: input.processing.optOutMeasures, illegal_content: input.processing.illegalContentMeasures },
|
|
46
|
+
};
|
|
47
|
+
return { ...summary, hash: jcsHash(summary) };
|
|
48
|
+
}
|
|
49
|
+
|
|
50
|
+
// ------------------------------------------------------------------------------ //
|
|
51
|
+
// Exclusions: opt-outs and evaluation sets
|
|
52
|
+
// ------------------------------------------------------------------------------ //
|
|
53
|
+
|
|
54
|
+
/** For every opted-out work: an absence proof, or a violation when it is in the set. */
|
|
55
|
+
export function optOutsHonoured(m: TrainingManifest, refs: string[], optedOut: string[]) {
|
|
56
|
+
const present = new Set(refs);
|
|
57
|
+
const proofs: AbsenceProof[] = [];
|
|
58
|
+
const violations: string[] = [];
|
|
59
|
+
for (const r of optedOut) present.has(r) ? violations.push(r) : proofs.push(proveAbsence(m, refs, r));
|
|
60
|
+
return { ok: violations.length === 0, proofs, violations };
|
|
61
|
+
}
|
|
62
|
+
|
|
63
|
+
/** The same check for an evaluation set: contamination is any eval item found in training. */
|
|
64
|
+
export function evalContamination(m: TrainingManifest, refs: string[], evalRefs: string[]) {
|
|
65
|
+
const r = optOutsHonoured(m, refs, evalRefs);
|
|
66
|
+
return { clean: r.ok, contaminated: r.violations, rate: evalRefs.length ? r.violations.length / evalRefs.length : 0, absenceProofs: r.proofs };
|
|
67
|
+
}
|
|
68
|
+
|
|
69
|
+
// ------------------------------------------------------------------------------ //
|
|
70
|
+
// Licences, lineage, attestations, fine-tunes
|
|
71
|
+
// ------------------------------------------------------------------------------ //
|
|
72
|
+
|
|
73
|
+
export interface SourceLicence {
|
|
74
|
+
source: string;
|
|
75
|
+
licence: string;
|
|
76
|
+
termsHash: string;
|
|
77
|
+
allowsTraining: boolean;
|
|
78
|
+
}
|
|
79
|
+
|
|
80
|
+
/** Commit per-source licence terms (as refs in a manifest) and list sources whose terms do not allow training. */
|
|
81
|
+
export function licenceLedger(runId: string, sources: SourceLicence[], createdAt: string) {
|
|
82
|
+
const refs = sources.map((s) => jcsHash(s as never));
|
|
83
|
+
return { manifest: buildManifest({ runId: `${runId}#licences`, refs, createdAt }), refs, notAllowed: sources.filter((s) => !s.allowsTraining).map((s) => s.source) };
|
|
84
|
+
}
|
|
85
|
+
|
|
86
|
+
/** Prove one source's licence entry is in the committed licence ledger. */
|
|
87
|
+
export function licenceProof(ledger: ReturnType<typeof licenceLedger>, source: SourceLicence): MembershipProof {
|
|
88
|
+
return proveMembership(ledger.manifest, ledger.refs, jcsHash(source as never));
|
|
89
|
+
}
|
|
90
|
+
|
|
91
|
+
/** Synthetic samples traced to the generator that made them and the samples it was seeded with. */
|
|
92
|
+
export interface SyntheticRecord {
|
|
93
|
+
ref: string;
|
|
94
|
+
generator: { model: string; version: string; manifest?: string };
|
|
95
|
+
seeds: string[];
|
|
96
|
+
}
|
|
97
|
+
|
|
98
|
+
/** How many generations of synthetic data a sample sits on (0 for real data); deep chains signal model-collapse risk. */
|
|
99
|
+
export function syntheticDepth(records: SyntheticRecord[], ref: string, maxDepth = 32): number {
|
|
100
|
+
const by = new Map(records.map((r) => [r.ref, r]));
|
|
101
|
+
const depth = (x: string, d: number): number => {
|
|
102
|
+
const r = by.get(x);
|
|
103
|
+
if (!r || d >= maxDepth) return 0;
|
|
104
|
+
return 1 + Math.max(0, ...r.seeds.map((s) => depth(s, d + 1)));
|
|
105
|
+
};
|
|
106
|
+
return depth(ref, 0);
|
|
107
|
+
}
|
|
108
|
+
|
|
109
|
+
/** Bind a manifest to an attested training job (TEE or GPU attestation): one hash covering both. */
|
|
110
|
+
export function runAttestation(m: TrainingManifest, att: { platform: string; attestationHash: string; imageHash: string; startedAt: string; endedAt: string }) {
|
|
111
|
+
const binding = { v: 1, type: 'training.run_attestation', manifest: manifestHash(m), ...att };
|
|
112
|
+
return { ...binding, hash: jcsHash(binding) };
|
|
113
|
+
}
|
|
114
|
+
|
|
115
|
+
export function verifyRunAttestation(b: ReturnType<typeof runAttestation>, m: TrainingManifest): boolean {
|
|
116
|
+
const { hash, ...rest } = b;
|
|
117
|
+
return rest.manifest === manifestHash(m) && jcsHash(rest) === hash;
|
|
118
|
+
}
|
|
119
|
+
|
|
120
|
+
/** A fine-tune's own manifest, linked to its base model's manifest. */
|
|
121
|
+
export function fineTuneManifest(base: TrainingManifest, input: { runId: string; refs: string[]; createdAt: string }) {
|
|
122
|
+
return { manifest: buildManifest(input), base: manifestHash(base), base_run: base.run_id };
|
|
123
|
+
}
|
|
124
|
+
|
|
125
|
+
// ------------------------------------------------------------------------------ //
|
|
126
|
+
// Royalties and forensics
|
|
127
|
+
// ------------------------------------------------------------------------------ //
|
|
128
|
+
|
|
129
|
+
/** Split a pool (minor units) among contributors by how many of their works were included. Exact; remainders by largest fraction. */
|
|
130
|
+
export function royaltyByInclusion(included: string[], contributorOf: Record<string, string>, poolMinor: bigint) {
|
|
131
|
+
const counts = new Map<string, bigint>();
|
|
132
|
+
for (const r of new Set(included)) {
|
|
133
|
+
const c = contributorOf[r];
|
|
134
|
+
if (c) counts.set(c, (counts.get(c) ?? 0n) + 1n);
|
|
135
|
+
}
|
|
136
|
+
const total = [...counts.values()].reduce((a, b) => a + b, 0n);
|
|
137
|
+
if (total === 0n) return { payouts: [] as Array<{ contributor: string; works: number; minor: string }> };
|
|
138
|
+
const rows = [...counts].map(([contributor, n]) => ({ contributor, n, q: (poolMinor * n) / total, r: (poolMinor * n) % total }));
|
|
139
|
+
let left = poolMinor - rows.reduce((s, x) => s + x.q, 0n);
|
|
140
|
+
for (const x of [...rows].sort((a, b) => (b.r > a.r ? 1 : b.r < a.r ? -1 : a.contributor.localeCompare(b.contributor)))) {
|
|
141
|
+
if (left === 0n) break;
|
|
142
|
+
x.q += 1n;
|
|
143
|
+
left -= 1n;
|
|
144
|
+
}
|
|
145
|
+
return { payouts: rows.map((x) => ({ contributor: x.contributor, works: Number(x.n), minor: x.q.toString() })) };
|
|
146
|
+
}
|
|
147
|
+
|
|
148
|
+
/** Which runs included a suspect sample: a membership proof for each, an absence proof for the rest. */
|
|
149
|
+
export function poisoningForensics(sampleRef: string, runs: Array<{ manifest: TrainingManifest; refs: string[] }>) {
|
|
150
|
+
const affected: Array<{ run: string; proof: MembershipProof }> = [];
|
|
151
|
+
const clear: Array<{ run: string; proof: AbsenceProof }> = [];
|
|
152
|
+
for (const r of runs) {
|
|
153
|
+
if (r.refs.includes(sampleRef)) affected.push({ run: r.manifest.run_id, proof: proveMembership(r.manifest, r.refs, sampleRef) });
|
|
154
|
+
else clear.push({ run: r.manifest.run_id, proof: proveAbsence(r.manifest, r.refs, sampleRef) });
|
|
155
|
+
}
|
|
156
|
+
return { sample: sampleRef, affected, clear };
|
|
157
|
+
}
|