@image-cascade/conformance 0.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/corpus/README.md +3 -0
- package/corpus/anthropic_document_positional_historical.json +61 -0
- package/corpus/anthropic_positional_historical.json +61 -0
- package/corpus/nested_boundary_regression.json +78 -0
- package/corpus/no_image_payload.json +27 -0
- package/corpus/openai_chat_positional_historical.json +57 -0
- package/corpus/openai_responses_positional_historical.json +53 -0
- package/corpus/tracker_current_hash_retained.json +34 -0
- package/dist/suite.d.ts +47 -0
- package/dist/suite.d.ts.map +1 -0
- package/dist/suite.js +81 -0
- package/dist/suite.js.map +1 -0
- package/package.json +50 -0
package/corpus/README.md
ADDED
|
@@ -0,0 +1,3 @@
|
|
|
1
|
+
# Image Context Cascade Corpus
|
|
2
|
+
|
|
3
|
+
Language-neutral JSON cases used by the conformance suite. Circular object graphs cannot be represented in JSON; the circular reference scenario is therefore covered by the hand-written `circular_reference_payload_no_stack_overflow` core test.
|
|
@@ -0,0 +1,61 @@
|
|
|
1
|
+
{
|
|
2
|
+
"name": "anthropic_document_positional_historical",
|
|
3
|
+
"description": "Anthropic base64 document historical block is downgraded and current document is retained",
|
|
4
|
+
"payload": {
|
|
5
|
+
"messages": [
|
|
6
|
+
{
|
|
7
|
+
"role": "user",
|
|
8
|
+
"content": [
|
|
9
|
+
{
|
|
10
|
+
"type": "text",
|
|
11
|
+
"text": "old document"
|
|
12
|
+
},
|
|
13
|
+
{
|
|
14
|
+
"type": "document",
|
|
15
|
+
"source": {
|
|
16
|
+
"type": "base64",
|
|
17
|
+
"media_type": "application/pdf",
|
|
18
|
+
"data": "Y29ycC1kb2Mtb2xk"
|
|
19
|
+
}
|
|
20
|
+
}
|
|
21
|
+
]
|
|
22
|
+
},
|
|
23
|
+
{
|
|
24
|
+
"role": "assistant",
|
|
25
|
+
"content": [
|
|
26
|
+
{
|
|
27
|
+
"type": "text",
|
|
28
|
+
"text": "seen"
|
|
29
|
+
}
|
|
30
|
+
]
|
|
31
|
+
},
|
|
32
|
+
{
|
|
33
|
+
"role": "user",
|
|
34
|
+
"content": [
|
|
35
|
+
{
|
|
36
|
+
"type": "text",
|
|
37
|
+
"text": "now"
|
|
38
|
+
},
|
|
39
|
+
{
|
|
40
|
+
"type": "document",
|
|
41
|
+
"source": {
|
|
42
|
+
"type": "base64",
|
|
43
|
+
"media_type": "application/pdf",
|
|
44
|
+
"data": "Y29ycC1kb2Mtbm93"
|
|
45
|
+
}
|
|
46
|
+
}
|
|
47
|
+
]
|
|
48
|
+
}
|
|
49
|
+
]
|
|
50
|
+
},
|
|
51
|
+
"options": {
|
|
52
|
+
"strategy": "positional"
|
|
53
|
+
},
|
|
54
|
+
"expect": {
|
|
55
|
+
"mutated": true,
|
|
56
|
+
"current": 1,
|
|
57
|
+
"downgraded": 1,
|
|
58
|
+
"unknownIntact": 0,
|
|
59
|
+
"downgradedPlaceholderStable": false
|
|
60
|
+
}
|
|
61
|
+
}
|
|
@@ -0,0 +1,61 @@
|
|
|
1
|
+
{
|
|
2
|
+
"name": "anthropic_positional_historical",
|
|
3
|
+
"description": "Anthropic historical image downgraded, current retained",
|
|
4
|
+
"payload": {
|
|
5
|
+
"messages": [
|
|
6
|
+
{
|
|
7
|
+
"role": "user",
|
|
8
|
+
"content": [
|
|
9
|
+
{
|
|
10
|
+
"type": "text",
|
|
11
|
+
"text": "old"
|
|
12
|
+
},
|
|
13
|
+
{
|
|
14
|
+
"type": "image",
|
|
15
|
+
"source": {
|
|
16
|
+
"type": "base64",
|
|
17
|
+
"media_type": "image/png",
|
|
18
|
+
"data": "Y29ycC1hbnRoLW9sZA=="
|
|
19
|
+
}
|
|
20
|
+
}
|
|
21
|
+
]
|
|
22
|
+
},
|
|
23
|
+
{
|
|
24
|
+
"role": "assistant",
|
|
25
|
+
"content": [
|
|
26
|
+
{
|
|
27
|
+
"type": "text",
|
|
28
|
+
"text": "ok"
|
|
29
|
+
}
|
|
30
|
+
]
|
|
31
|
+
},
|
|
32
|
+
{
|
|
33
|
+
"role": "user",
|
|
34
|
+
"content": [
|
|
35
|
+
{
|
|
36
|
+
"type": "text",
|
|
37
|
+
"text": "now"
|
|
38
|
+
},
|
|
39
|
+
{
|
|
40
|
+
"type": "image",
|
|
41
|
+
"source": {
|
|
42
|
+
"type": "base64",
|
|
43
|
+
"media_type": "image/png",
|
|
44
|
+
"data": "Y29ycC1hbnRoLW5vdw=="
|
|
45
|
+
}
|
|
46
|
+
}
|
|
47
|
+
]
|
|
48
|
+
}
|
|
49
|
+
]
|
|
50
|
+
},
|
|
51
|
+
"options": {
|
|
52
|
+
"strategy": "positional"
|
|
53
|
+
},
|
|
54
|
+
"expect": {
|
|
55
|
+
"mutated": true,
|
|
56
|
+
"current": 1,
|
|
57
|
+
"downgraded": 1,
|
|
58
|
+
"unknownIntact": 0,
|
|
59
|
+
"downgradedPlaceholderStable": true
|
|
60
|
+
}
|
|
61
|
+
}
|
|
@@ -0,0 +1,78 @@
|
|
|
1
|
+
{
|
|
2
|
+
"name": "nested_boundary_regression",
|
|
3
|
+
"description": "Nested transcript must not move top-level current boundary",
|
|
4
|
+
"payload": {
|
|
5
|
+
"messages": [
|
|
6
|
+
{
|
|
7
|
+
"role": "user",
|
|
8
|
+
"content": [
|
|
9
|
+
{
|
|
10
|
+
"type": "text",
|
|
11
|
+
"text": "old"
|
|
12
|
+
},
|
|
13
|
+
{
|
|
14
|
+
"type": "image",
|
|
15
|
+
"source": {
|
|
16
|
+
"type": "base64",
|
|
17
|
+
"media_type": "image/png",
|
|
18
|
+
"data": "bmVzdGVkLW9sZA=="
|
|
19
|
+
}
|
|
20
|
+
}
|
|
21
|
+
]
|
|
22
|
+
},
|
|
23
|
+
{
|
|
24
|
+
"role": "assistant",
|
|
25
|
+
"content": [
|
|
26
|
+
{
|
|
27
|
+
"type": "text",
|
|
28
|
+
"text": "tool"
|
|
29
|
+
},
|
|
30
|
+
{
|
|
31
|
+
"transcript": [
|
|
32
|
+
{
|
|
33
|
+
"role": "user",
|
|
34
|
+
"content": [
|
|
35
|
+
{
|
|
36
|
+
"type": "image",
|
|
37
|
+
"source": {
|
|
38
|
+
"type": "base64",
|
|
39
|
+
"media_type": "image/png",
|
|
40
|
+
"data": "bmVzdGVkLWlubmVyLWN1cnJlbnQ="
|
|
41
|
+
}
|
|
42
|
+
}
|
|
43
|
+
]
|
|
44
|
+
}
|
|
45
|
+
]
|
|
46
|
+
}
|
|
47
|
+
]
|
|
48
|
+
},
|
|
49
|
+
{
|
|
50
|
+
"role": "user",
|
|
51
|
+
"content": [
|
|
52
|
+
{
|
|
53
|
+
"type": "text",
|
|
54
|
+
"text": "now"
|
|
55
|
+
},
|
|
56
|
+
{
|
|
57
|
+
"type": "image",
|
|
58
|
+
"source": {
|
|
59
|
+
"type": "base64",
|
|
60
|
+
"media_type": "image/png",
|
|
61
|
+
"data": "bmVzdGVkLW5vdw=="
|
|
62
|
+
}
|
|
63
|
+
}
|
|
64
|
+
]
|
|
65
|
+
}
|
|
66
|
+
]
|
|
67
|
+
},
|
|
68
|
+
"options": {
|
|
69
|
+
"strategy": "positional"
|
|
70
|
+
},
|
|
71
|
+
"expect": {
|
|
72
|
+
"mutated": true,
|
|
73
|
+
"current": 2,
|
|
74
|
+
"downgraded": 1,
|
|
75
|
+
"unknownIntact": 0,
|
|
76
|
+
"downgradedPlaceholderStable": true
|
|
77
|
+
}
|
|
78
|
+
}
|
|
@@ -0,0 +1,27 @@
|
|
|
1
|
+
{
|
|
2
|
+
"name": "no_image_payload",
|
|
3
|
+
"description": "Payload without image blocks remains untouched",
|
|
4
|
+
"payload": {
|
|
5
|
+
"messages": [
|
|
6
|
+
{
|
|
7
|
+
"role": "user",
|
|
8
|
+
"content": [
|
|
9
|
+
{
|
|
10
|
+
"type": "text",
|
|
11
|
+
"text": "hello"
|
|
12
|
+
}
|
|
13
|
+
]
|
|
14
|
+
}
|
|
15
|
+
]
|
|
16
|
+
},
|
|
17
|
+
"options": {
|
|
18
|
+
"strategy": "positional"
|
|
19
|
+
},
|
|
20
|
+
"expect": {
|
|
21
|
+
"mutated": false,
|
|
22
|
+
"current": 0,
|
|
23
|
+
"downgraded": 0,
|
|
24
|
+
"unknownIntact": 0,
|
|
25
|
+
"downgradedPlaceholderStable": true
|
|
26
|
+
}
|
|
27
|
+
}
|
|
@@ -0,0 +1,57 @@
|
|
|
1
|
+
{
|
|
2
|
+
"name": "openai_chat_positional_historical",
|
|
3
|
+
"description": "OpenAI chat image_url object downgraded",
|
|
4
|
+
"payload": {
|
|
5
|
+
"messages": [
|
|
6
|
+
{
|
|
7
|
+
"role": "user",
|
|
8
|
+
"content": [
|
|
9
|
+
{
|
|
10
|
+
"type": "text",
|
|
11
|
+
"text": "old"
|
|
12
|
+
},
|
|
13
|
+
{
|
|
14
|
+
"type": "image_url",
|
|
15
|
+
"image_url": {
|
|
16
|
+
"url": "data:image/png;base64,Y29ycC1jaGF0LW9sZA=="
|
|
17
|
+
}
|
|
18
|
+
}
|
|
19
|
+
]
|
|
20
|
+
},
|
|
21
|
+
{
|
|
22
|
+
"role": "assistant",
|
|
23
|
+
"content": [
|
|
24
|
+
{
|
|
25
|
+
"type": "text",
|
|
26
|
+
"text": "ok"
|
|
27
|
+
}
|
|
28
|
+
]
|
|
29
|
+
},
|
|
30
|
+
{
|
|
31
|
+
"role": "user",
|
|
32
|
+
"content": [
|
|
33
|
+
{
|
|
34
|
+
"type": "text",
|
|
35
|
+
"text": "now"
|
|
36
|
+
},
|
|
37
|
+
{
|
|
38
|
+
"type": "image_url",
|
|
39
|
+
"image_url": {
|
|
40
|
+
"url": "data:image/png;base64,Y29ycC1jaGF0LW5vdw=="
|
|
41
|
+
}
|
|
42
|
+
}
|
|
43
|
+
]
|
|
44
|
+
}
|
|
45
|
+
]
|
|
46
|
+
},
|
|
47
|
+
"options": {
|
|
48
|
+
"strategy": "positional"
|
|
49
|
+
},
|
|
50
|
+
"expect": {
|
|
51
|
+
"mutated": true,
|
|
52
|
+
"current": 1,
|
|
53
|
+
"downgraded": 1,
|
|
54
|
+
"unknownIntact": 0,
|
|
55
|
+
"downgradedPlaceholderStable": true
|
|
56
|
+
}
|
|
57
|
+
}
|
|
@@ -0,0 +1,53 @@
|
|
|
1
|
+
{
|
|
2
|
+
"name": "openai_responses_positional_historical",
|
|
3
|
+
"description": "OpenAI responses input_image downgraded",
|
|
4
|
+
"payload": {
|
|
5
|
+
"messages": [
|
|
6
|
+
{
|
|
7
|
+
"role": "user",
|
|
8
|
+
"content": [
|
|
9
|
+
{
|
|
10
|
+
"type": "text",
|
|
11
|
+
"text": "old"
|
|
12
|
+
},
|
|
13
|
+
{
|
|
14
|
+
"type": "input_image",
|
|
15
|
+
"image_url": "data:image/png;base64,Y29ycC1yZXNwLW9sZA=="
|
|
16
|
+
}
|
|
17
|
+
]
|
|
18
|
+
},
|
|
19
|
+
{
|
|
20
|
+
"role": "assistant",
|
|
21
|
+
"content": [
|
|
22
|
+
{
|
|
23
|
+
"type": "text",
|
|
24
|
+
"text": "ok"
|
|
25
|
+
}
|
|
26
|
+
]
|
|
27
|
+
},
|
|
28
|
+
{
|
|
29
|
+
"role": "user",
|
|
30
|
+
"content": [
|
|
31
|
+
{
|
|
32
|
+
"type": "text",
|
|
33
|
+
"text": "now"
|
|
34
|
+
},
|
|
35
|
+
{
|
|
36
|
+
"type": "input_image",
|
|
37
|
+
"image_url": "data:image/png;base64,Y29ycC1yZXNwLW5vdw=="
|
|
38
|
+
}
|
|
39
|
+
]
|
|
40
|
+
}
|
|
41
|
+
]
|
|
42
|
+
},
|
|
43
|
+
"options": {
|
|
44
|
+
"strategy": "positional"
|
|
45
|
+
},
|
|
46
|
+
"expect": {
|
|
47
|
+
"mutated": true,
|
|
48
|
+
"current": 1,
|
|
49
|
+
"downgraded": 1,
|
|
50
|
+
"unknownIntact": 0,
|
|
51
|
+
"downgradedPlaceholderStable": true
|
|
52
|
+
}
|
|
53
|
+
}
|
|
@@ -0,0 +1,34 @@
|
|
|
1
|
+
{
|
|
2
|
+
"name": "tracker_current_hash_retained",
|
|
3
|
+
"description": "Tracker strategy retains explicitly current image",
|
|
4
|
+
"payload": {
|
|
5
|
+
"messages": [
|
|
6
|
+
{
|
|
7
|
+
"role": "user",
|
|
8
|
+
"content": [
|
|
9
|
+
{
|
|
10
|
+
"type": "image",
|
|
11
|
+
"source": {
|
|
12
|
+
"type": "base64",
|
|
13
|
+
"media_type": "image/png",
|
|
14
|
+
"data": "dHJhY2tlci1jdXJyZW50"
|
|
15
|
+
}
|
|
16
|
+
}
|
|
17
|
+
]
|
|
18
|
+
}
|
|
19
|
+
]
|
|
20
|
+
},
|
|
21
|
+
"options": {
|
|
22
|
+
"strategy": "tracker",
|
|
23
|
+
"currentHashesOf": [
|
|
24
|
+
"messages.0.content.0"
|
|
25
|
+
]
|
|
26
|
+
},
|
|
27
|
+
"expect": {
|
|
28
|
+
"mutated": false,
|
|
29
|
+
"current": 1,
|
|
30
|
+
"downgraded": 0,
|
|
31
|
+
"unknownIntact": 0,
|
|
32
|
+
"downgradedPlaceholderStable": true
|
|
33
|
+
}
|
|
34
|
+
}
|
package/dist/suite.d.ts
ADDED
|
@@ -0,0 +1,47 @@
|
|
|
1
|
+
export interface AdapterHarness {
|
|
2
|
+
name: string;
|
|
3
|
+
runTwoTurnImageRequest(payloads: {
|
|
4
|
+
first: unknown;
|
|
5
|
+
second: unknown;
|
|
6
|
+
}): Promise<{
|
|
7
|
+
firstPayload: unknown;
|
|
8
|
+
secondPayload: unknown;
|
|
9
|
+
telemetry: unknown[];
|
|
10
|
+
}> | {
|
|
11
|
+
firstPayload: unknown;
|
|
12
|
+
secondPayload: unknown;
|
|
13
|
+
telemetry: unknown[];
|
|
14
|
+
};
|
|
15
|
+
}
|
|
16
|
+
export interface ConformanceResult {
|
|
17
|
+
adapterName: string;
|
|
18
|
+
passed: boolean;
|
|
19
|
+
checks: string[];
|
|
20
|
+
}
|
|
21
|
+
export interface CorpusRunResult {
|
|
22
|
+
passed: boolean;
|
|
23
|
+
checks: string[];
|
|
24
|
+
cases: number;
|
|
25
|
+
}
|
|
26
|
+
type CorpusCase = {
|
|
27
|
+
name: string;
|
|
28
|
+
description: string;
|
|
29
|
+
payload: unknown;
|
|
30
|
+
options?: {
|
|
31
|
+
strategy?: "positional" | "tracker";
|
|
32
|
+
currentHashesOf?: string[];
|
|
33
|
+
};
|
|
34
|
+
expect: {
|
|
35
|
+
mutated: boolean;
|
|
36
|
+
current: number;
|
|
37
|
+
downgraded: number;
|
|
38
|
+
unknownIntact: number;
|
|
39
|
+
downgradedPlaceholderStable: boolean;
|
|
40
|
+
};
|
|
41
|
+
};
|
|
42
|
+
export declare function loadCorpusCases(): CorpusCase[];
|
|
43
|
+
export declare function runCorpusConformance(): CorpusRunResult;
|
|
44
|
+
export declare function runConformance(adapterHarness: AdapterHarness): Promise<ConformanceResult>;
|
|
45
|
+
export declare function referenceCoreHarness(): AdapterHarness;
|
|
46
|
+
export {};
|
|
47
|
+
//# sourceMappingURL=suite.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"suite.d.ts","sourceRoot":"","sources":["../src/suite.ts"],"names":[],"mappings":"AAKA,MAAM,WAAW,cAAc;IAC7B,IAAI,EAAE,MAAM,CAAC;IACb,sBAAsB,CAAC,QAAQ,EAAE;QAAE,KAAK,EAAE,OAAO,CAAC;QAAC,MAAM,EAAE,OAAO,CAAA;KAAE,GAAG,OAAO,CAAC;QAC7E,YAAY,EAAE,OAAO,CAAC;QACtB,aAAa,EAAE,OAAO,CAAC;QACvB,SAAS,EAAE,OAAO,EAAE,CAAC;KACtB,CAAC,GAAG;QACH,YAAY,EAAE,OAAO,CAAC;QACtB,aAAa,EAAE,OAAO,CAAC;QACvB,SAAS,EAAE,OAAO,EAAE,CAAC;KACtB,CAAC;CACH;AAED,MAAM,WAAW,iBAAiB;IAChC,WAAW,EAAE,MAAM,CAAC;IACpB,MAAM,EAAE,OAAO,CAAC;IAChB,MAAM,EAAE,MAAM,EAAE,CAAC;CAClB;AAED,MAAM,WAAW,eAAe;IAC9B,MAAM,EAAE,OAAO,CAAC;IAChB,MAAM,EAAE,MAAM,EAAE,CAAC;IACjB,KAAK,EAAE,MAAM,CAAC;CACf;AAED,KAAK,UAAU,GAAG;IAChB,IAAI,EAAE,MAAM,CAAC;IACb,WAAW,EAAE,MAAM,CAAC;IACpB,OAAO,EAAE,OAAO,CAAC;IACjB,OAAO,CAAC,EAAE;QAAE,QAAQ,CAAC,EAAE,YAAY,GAAG,SAAS,CAAC;QAAC,eAAe,CAAC,EAAE,MAAM,EAAE,CAAA;KAAE,CAAC;IAC9E,MAAM,EAAE;QAAE,OAAO,EAAE,OAAO,CAAC;QAAC,OAAO,EAAE,MAAM,CAAC;QAAC,UAAU,EAAE,MAAM,CAAC;QAAC,aAAa,EAAE,MAAM,CAAC;QAAC,2BAA2B,EAAE,OAAO,CAAA;KAAE,CAAC;CAChI,CAAC;AAwBF,wBAAgB,eAAe,IAAI,UAAU,EAAE,CAK9C;AAED,wBAAgB,oBAAoB,IAAI,eAAe,CAiBtD;AAED,wBAAsB,cAAc,CAAC,cAAc,EAAE,cAAc,GAAG,OAAO,CAAC,iBAAiB,CAAC,CAgB/F;AAED,wBAAgB,oBAAoB,IAAI,cAAc,CASrD"}
|
package/dist/suite.js
ADDED
|
@@ -0,0 +1,81 @@
|
|
|
1
|
+
import { readdirSync, readFileSync } from "node:fs";
|
|
2
|
+
import { dirname, join } from "node:path";
|
|
3
|
+
import { fileURLToPath } from "node:url";
|
|
4
|
+
import { cascadeImages, defaultPlaceholder, imageIdentity, InMemoryTracker, trackerStrategy } from "image-context-cascade";
|
|
5
|
+
function getPath(root, path) {
|
|
6
|
+
return path.split(".").reduce((value, part) => {
|
|
7
|
+
if (value == null)
|
|
8
|
+
return undefined;
|
|
9
|
+
if (Array.isArray(value))
|
|
10
|
+
return value[Number(part)];
|
|
11
|
+
if (typeof value === "object")
|
|
12
|
+
return value[part];
|
|
13
|
+
return undefined;
|
|
14
|
+
}, root);
|
|
15
|
+
}
|
|
16
|
+
function imageDataFromBlock(block) {
|
|
17
|
+
if (!block || typeof block !== "object")
|
|
18
|
+
throw new Error("block path did not resolve to an object");
|
|
19
|
+
const b = block;
|
|
20
|
+
if (b.type === "image" && typeof b.source?.data === "string")
|
|
21
|
+
return b.source.data;
|
|
22
|
+
const raw = typeof b.image_url === "string" ? b.image_url : (b.image_url && typeof b.image_url === "object" ? b.image_url.url : undefined);
|
|
23
|
+
if (typeof raw === "string")
|
|
24
|
+
return raw.startsWith("data:") ? raw.split(",", 2)[1] ?? raw : raw;
|
|
25
|
+
throw new Error("unsupported block in currentHashesOf");
|
|
26
|
+
}
|
|
27
|
+
function corpusDir() {
|
|
28
|
+
return join(dirname(fileURLToPath(import.meta.url)), "..", "corpus");
|
|
29
|
+
}
|
|
30
|
+
export function loadCorpusCases() {
|
|
31
|
+
return readdirSync(corpusDir())
|
|
32
|
+
.filter((name) => name.endsWith(".json"))
|
|
33
|
+
.sort()
|
|
34
|
+
.map((name) => JSON.parse(readFileSync(join(corpusDir(), name), "utf8")));
|
|
35
|
+
}
|
|
36
|
+
export function runCorpusConformance() {
|
|
37
|
+
const checks = [];
|
|
38
|
+
for (const c of loadCorpusCases()) {
|
|
39
|
+
const options = c.options?.strategy === "tracker"
|
|
40
|
+
? { strategy: trackerStrategy({ tracker: new InMemoryTracker(), currentTurnHashes: new Set((c.options.currentHashesOf ?? []).map((path) => imageIdentity(imageDataFromBlock(getPath(c.payload, path))).hash)) }) }
|
|
41
|
+
: {};
|
|
42
|
+
const result = cascadeImages(c.payload, options);
|
|
43
|
+
const t = result.telemetry;
|
|
44
|
+
const ok = result.mutated === c.expect.mutated && t.current === c.expect.current && t.downgraded === c.expect.downgraded && t.unknownIntact === c.expect.unknownIntact;
|
|
45
|
+
checks.push(ok ? `${c.name}: ok` : `${c.name}: failed`);
|
|
46
|
+
if (c.expect.downgradedPlaceholderStable && t.shortHashes.length > 0 && t.downgraded > 0) {
|
|
47
|
+
const json = JSON.stringify(result.payload);
|
|
48
|
+
const containsKnownPlaceholder = t.shortHashes.some((shortHash) => json.includes(defaultPlaceholder({ hash: shortHash, shortHash })));
|
|
49
|
+
checks.push(containsKnownPlaceholder ? `${c.name}: stable placeholder` : `${c.name}: missing stable placeholder`);
|
|
50
|
+
}
|
|
51
|
+
}
|
|
52
|
+
return { passed: checks.every((c) => c.endsWith(": ok") || c.endsWith(": stable placeholder")), checks, cases: loadCorpusCases().length };
|
|
53
|
+
}
|
|
54
|
+
export async function runConformance(adapterHarness) {
|
|
55
|
+
const data = Buffer.from("synthetic-conformance-image").toString("base64");
|
|
56
|
+
const first = { messages: [{ role: "user", content: [{ type: "input_image", image_url: `data:image/png;base64,${data}` }] }] };
|
|
57
|
+
const second = { messages: [
|
|
58
|
+
{ role: "user", content: [{ type: "input_image", image_url: `data:image/png;base64,${data}` }] },
|
|
59
|
+
{ role: "assistant", content: [{ type: "text", text: "seen" }] },
|
|
60
|
+
{ role: "user", content: [{ type: "text", text: "continue" }] },
|
|
61
|
+
] };
|
|
62
|
+
const result = await adapterHarness.runTwoTurnImageRequest({ first, second });
|
|
63
|
+
const jsonTelemetry = JSON.stringify(result.telemetry);
|
|
64
|
+
const checks = [
|
|
65
|
+
JSON.stringify(result.firstPayload).includes("input_image") ? "current retained" : "missing current retained",
|
|
66
|
+
JSON.stringify(result.secondPayload).includes(defaultPlaceholder(imageIdentity(data))) ? "historical downgraded" : "missing historical downgrade",
|
|
67
|
+
!jsonTelemetry.includes("data:image") ? "telemetry privacy" : "telemetry leaked data uri",
|
|
68
|
+
];
|
|
69
|
+
return { adapterName: adapterHarness.name, passed: checks.every((c) => !c.startsWith("missing") && !c.includes("leaked")), checks };
|
|
70
|
+
}
|
|
71
|
+
export function referenceCoreHarness() {
|
|
72
|
+
return {
|
|
73
|
+
name: "core-reference",
|
|
74
|
+
runTwoTurnImageRequest({ first, second }) {
|
|
75
|
+
const r1 = cascadeImages(first);
|
|
76
|
+
const r2 = cascadeImages(second);
|
|
77
|
+
return { firstPayload: r1.payload, secondPayload: r2.payload, telemetry: [r1.telemetry, r2.telemetry] };
|
|
78
|
+
},
|
|
79
|
+
};
|
|
80
|
+
}
|
|
81
|
+
//# sourceMappingURL=suite.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"suite.js","sourceRoot":"","sources":["../src/suite.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,WAAW,EAAE,YAAY,EAAE,MAAM,SAAS,CAAC;AACpD,OAAO,EAAE,OAAO,EAAE,IAAI,EAAE,MAAM,WAAW,CAAC;AAC1C,OAAO,EAAE,aAAa,EAAE,MAAM,UAAU,CAAC;AACzC,OAAO,EAAE,aAAa,EAAE,kBAAkB,EAAE,aAAa,EAAE,eAAe,EAAE,eAAe,EAAyB,MAAM,uBAAuB,CAAC;AAmClJ,SAAS,OAAO,CAAC,IAAa,EAAE,IAAY;IAC1C,OAAO,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,MAAM,CAAC,CAAC,KAAc,EAAE,IAAI,EAAE,EAAE;QACrD,IAAI,KAAK,IAAI,IAAI;YAAE,OAAO,SAAS,CAAC;QACpC,IAAI,KAAK,CAAC,OAAO,CAAC,KAAK,CAAC;YAAE,OAAO,KAAK,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC,CAAC;QACrD,IAAI,OAAO,KAAK,KAAK,QAAQ;YAAE,OAAQ,KAAiC,CAAC,IAAI,CAAC,CAAC;QAC/E,OAAO,SAAS,CAAC;IACnB,CAAC,EAAE,IAAI,CAAC,CAAC;AACX,CAAC;AAED,SAAS,kBAAkB,CAAC,KAAc;IACxC,IAAI,CAAC,KAAK,IAAI,OAAO,KAAK,KAAK,QAAQ;QAAE,MAAM,IAAI,KAAK,CAAC,yCAAyC,CAAC,CAAC;IACpG,MAAM,CAAC,GAAG,KAA6E,CAAC;IACxF,IAAI,CAAC,CAAC,IAAI,KAAK,OAAO,IAAI,OAAO,CAAC,CAAC,MAAM,EAAE,IAAI,KAAK,QAAQ;QAAE,OAAO,CAAC,CAAC,MAAM,CAAC,IAAI,CAAC;IACnF,MAAM,GAAG,GAAG,OAAO,CAAC,CAAC,SAAS,KAAK,QAAQ,CAAC,CAAC,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,SAAS,IAAI,OAAO,CAAC,CAAC,SAAS,KAAK,QAAQ,CAAC,CAAC,CAAE,CAAC,CAAC,SAA+B,CAAC,GAAG,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC;IAClK,IAAI,OAAO,GAAG,KAAK,QAAQ;QAAE,OAAO,GAAG,CAAC,UAAU,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,KAAK,CAAC,GAAG,EAAE,CAAC,CAAC,CAAC,CAAC,CAAC,IAAI,GAAG,CAAC,CAAC,CAAC,GAAG,CAAC;IAChG,MAAM,IAAI,KAAK,CAAC,sCAAsC,CAAC,CAAC;AAC1D,CAAC;AAED,SAAS,SAAS;IAChB,OAAO,IAAI,CAAC,OAAO,CAAC,aAAa,CAAC,MAAM,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,IAAI,EAAE,QAAQ,CAAC,CAAC;AACvE,CAAC;AAED,MAAM,UAAU,eAAe;IAC7B,OAAO,WAAW,CAAC,SAAS,EAAE,CAAC;SAC5B,MAAM,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,IAAI,CAAC,QAAQ,CAAC,OAAO,CAAC,CAAC;SACxC,IAAI,EAAE;SACN,GAAG,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,IAAI,CAAC,KAAK,CAAC,YAAY,CAAC,IAAI,CAAC,SAAS,EAAE,EAAE,IAAI,CAAC,EAAE,MAAM,CAAC,CAAe,CAAC,CAAC;AAC5F,CAAC;AAED,MAAM,UAAU,oBAAoB;IAClC,MAAM,MAAM,GAAa,EAAE,CAAC;IAC5B,KAAK,MAAM,CAAC,IAAI,eAAe,EAAE,EAAE,CAAC;QAClC,MAAM,OAAO,GAAG,CAAC,CAAC,OAAO,EAAE,QAAQ,KAAK,SAAS;YAC/C,CAAC,CAAC,EAAE,QAAQ,EAAE,eAAe,CAAC,EAAE,OAAO,EAAE,IAAI,eAAe,EAAE,EAAE,iBAAiB,EAAE,IAAI,GAAG,CAAC,CAAC,CAAC,CAAC,OAAO,CAAC,eAAe,IAAI,EAAE,CAAC,CAAC,GAAG,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,aAAa,CAAC,kBAAkB,CAAC,OAAO,CAAC,CAAC,CAAC,OAAO,EAAE,IAAI,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,EAAE,CAAC,EAAE;YAClN,CAAC,CAAC,EAAE,CAAC;QACP,MAAM,MAAM,GAAG,aAAa,CAAC,CAAC,CAAC,OAAO,EAAE,OAAO,CAAC,CAAC;QACjD,MAAM,CAAC,GAAG,MAAM,CAAC,SAA6B,CAAC;QAC/C,MAAM,EAAE,GAAG,MAAM,CAAC,OAAO,KAAK,CAAC,CAAC,MAAM,CAAC,OAAO,IAAI,CAAC,CAAC,OAAO,KAAK,CAAC,CAAC,MAAM,CAAC,OAAO,IAAI,CAAC,CAAC,UAAU,KAAK,CAAC,CAAC,MAAM,CAAC,UAAU,IAAI,CAAC,CAAC,aAAa,KAAK,CAAC,CAAC,MAAM,CAAC,aAAa,CAAC;QACvK,MAAM,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,IAAI,MAAM,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,IAAI,UAAU,CAAC,CAAC;QACxD,IAAI,CAAC,CAAC,MAAM,CAAC,2BAA2B,IAAI,CAAC,CAAC,WAAW,CAAC,MAAM,GAAG,CAAC,IAAI,CAAC,CAAC,UAAU,GAAG,CAAC,EAAE,CAAC;YACzF,MAAM,IAAI,GAAG,IAAI,CAAC,SAAS,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC;YAC5C,MAAM,wBAAwB,GAAG,CAAC,CAAC,WAAW,CAAC,IAAI,CAAC,CAAC,SAAS,EAAE,EAAE,CAAC,IAAI,CAAC,QAAQ,CAAC,kBAAkB,CAAC,EAAE,IAAI,EAAE,SAAS,EAAE,SAAS,EAAE,CAAC,CAAC,CAAC,CAAC;YACtI,MAAM,CAAC,IAAI,CAAC,wBAAwB,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,IAAI,sBAAsB,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,IAAI,8BAA8B,CAAC,CAAC;QACpH,CAAC;IACH,CAAC;IACD,OAAO,EAAE,MAAM,EAAE,MAAM,CAAC,KAAK,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,QAAQ,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC,QAAQ,CAAC,sBAAsB,CAAC,CAAC,EAAE,MAAM,EAAE,KAAK,EAAE,eAAe,EAAE,CAAC,MAAM,EAAE,CAAC;AAC5I,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,cAAc,CAAC,cAA8B;IACjE,MAAM,IAAI,GAAG,MAAM,CAAC,IAAI,CAAC,6BAA6B,CAAC,CAAC,QAAQ,CAAC,QAAQ,CAAC,CAAC;IAC3E,MAAM,KAAK,GAAG,EAAE,QAAQ,EAAE,CAAC,EAAE,IAAI,EAAE,MAAM,EAAE,OAAO,EAAE,CAAC,EAAE,IAAI,EAAE,aAAa,EAAE,SAAS,EAAE,yBAAyB,IAAI,EAAE,EAAE,CAAC,EAAE,CAAC,EAAE,CAAC;IAC/H,MAAM,MAAM,GAAG,EAAE,QAAQ,EAAE;YACzB,EAAE,IAAI,EAAE,MAAM,EAAE,OAAO,EAAE,CAAC,EAAE,IAAI,EAAE,aAAa,EAAE,SAAS,EAAE,yBAAyB,IAAI,EAAE,EAAE,CAAC,EAAE;YAChG,EAAE,IAAI,EAAE,WAAW,EAAE,OAAO,EAAE,CAAC,EAAE,IAAI,EAAE,MAAM,EAAE,IAAI,EAAE,MAAM,EAAE,CAAC,EAAE;YAChE,EAAE,IAAI,EAAE,MAAM,EAAE,OAAO,EAAE,CAAC,EAAE,IAAI,EAAE,MAAM,EAAE,IAAI,EAAE,UAAU,EAAE,CAAC,EAAE;SAChE,EAAE,CAAC;IACJ,MAAM,MAAM,GAAG,MAAM,cAAc,CAAC,sBAAsB,CAAC,EAAE,KAAK,EAAE,MAAM,EAAE,CAAC,CAAC;IAC9E,MAAM,aAAa,GAAG,IAAI,CAAC,SAAS,CAAC,MAAM,CAAC,SAAS,CAAC,CAAC;IACvD,MAAM,MAAM,GAAG;QACb,IAAI,CAAC,SAAS,CAAC,MAAM,CAAC,YAAY,CAAC,CAAC,QAAQ,CAAC,aAAa,CAAC,CAAC,CAAC,CAAC,kBAAkB,CAAC,CAAC,CAAC,0BAA0B;QAC7G,IAAI,CAAC,SAAS,CAAC,MAAM,CAAC,aAAa,CAAC,CAAC,QAAQ,CAAC,kBAAkB,CAAC,aAAa,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,uBAAuB,CAAC,CAAC,CAAC,8BAA8B;QACjJ,CAAC,aAAa,CAAC,QAAQ,CAAC,YAAY,CAAC,CAAC,CAAC,CAAC,mBAAmB,CAAC,CAAC,CAAC,2BAA2B;KAC1F,CAAC;IACF,OAAO,EAAE,WAAW,EAAE,cAAc,CAAC,IAAI,EAAE,MAAM,EAAE,MAAM,CAAC,KAAK,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,CAAC,UAAU,CAAC,SAAS,CAAC,IAAI,CAAC,CAAC,CAAC,QAAQ,CAAC,QAAQ,CAAC,CAAC,EAAE,MAAM,EAAE,CAAC;AACtI,CAAC;AAED,MAAM,UAAU,oBAAoB;IAClC,OAAO;QACL,IAAI,EAAE,gBAAgB;QACtB,sBAAsB,CAAC,EAAE,KAAK,EAAE,MAAM,EAAE;YACtC,MAAM,EAAE,GAAG,aAAa,CAAC,KAAK,CAAC,CAAC;YAChC,MAAM,EAAE,GAAG,aAAa,CAAC,MAAM,CAAC,CAAC;YACjC,OAAO,EAAE,YAAY,EAAE,EAAE,CAAC,OAAO,EAAE,aAAa,EAAE,EAAE,CAAC,OAAO,EAAE,SAAS,EAAE,CAAC,EAAE,CAAC,SAAS,EAAE,EAAE,CAAC,SAAS,CAAC,EAAE,CAAC;QAC1G,CAAC;KACF,CAAC;AACJ,CAAC"}
|
package/package.json
ADDED
|
@@ -0,0 +1,50 @@
|
|
|
1
|
+
{
|
|
2
|
+
"name": "@image-cascade/conformance",
|
|
3
|
+
"version": "0.1.0",
|
|
4
|
+
"type": "module",
|
|
5
|
+
"license": "Apache-2.0",
|
|
6
|
+
"description": "Conformance suite and corpus for image-context-cascade adapters.",
|
|
7
|
+
"main": "./dist/suite.js",
|
|
8
|
+
"types": "./dist/suite.d.ts",
|
|
9
|
+
"exports": {
|
|
10
|
+
".": {
|
|
11
|
+
"types": "./dist/suite.d.ts",
|
|
12
|
+
"default": "./dist/suite.js"
|
|
13
|
+
},
|
|
14
|
+
"./corpus/*": "./corpus/*"
|
|
15
|
+
},
|
|
16
|
+
"files": [
|
|
17
|
+
"dist",
|
|
18
|
+
"corpus"
|
|
19
|
+
],
|
|
20
|
+
"publishConfig": {
|
|
21
|
+
"access": "public",
|
|
22
|
+
"registry": "https://registry.npmjs.org/"
|
|
23
|
+
},
|
|
24
|
+
"sideEffects": false,
|
|
25
|
+
"engines": {
|
|
26
|
+
"node": ">=18"
|
|
27
|
+
},
|
|
28
|
+
"repository": {
|
|
29
|
+
"type": "git",
|
|
30
|
+
"url": "git+https://github.com/dlgod7/image-context-cascade.git",
|
|
31
|
+
"directory": "packages/conformance"
|
|
32
|
+
},
|
|
33
|
+
"homepage": "https://github.com/dlgod7/image-context-cascade#readme",
|
|
34
|
+
"bugs": {
|
|
35
|
+
"url": "https://github.com/dlgod7/image-context-cascade/issues"
|
|
36
|
+
},
|
|
37
|
+
"keywords": [
|
|
38
|
+
"llm",
|
|
39
|
+
"agent",
|
|
40
|
+
"context",
|
|
41
|
+
"image",
|
|
42
|
+
"middleware",
|
|
43
|
+
"conformance",
|
|
44
|
+
"corpus"
|
|
45
|
+
],
|
|
46
|
+
"dependencies": {
|
|
47
|
+
"image-context-cascade": "0.1.0"
|
|
48
|
+
},
|
|
49
|
+
"devDependencies": {}
|
|
50
|
+
}
|