@carbonenginejs/runtime-resource 0.12.2 → 0.14.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (238) hide show
  1. package/README.md +21 -0
  2. package/dist/CjsResMan.js +393 -29
  3. package/dist/CjsResMan.js.map +1 -1
  4. package/dist/format/carbonEffect/CjsCarbonEffectReader.js +165 -118
  5. package/dist/format/carbonEffect/CjsCarbonEffectReader.js.map +1 -1
  6. package/dist/format/carbonEffect/buildCarbonEffectContainer.js +182 -0
  7. package/dist/format/carbonEffect/buildCarbonEffectContainer.js.map +1 -0
  8. package/dist/format/carbonEffect/carbonEffectBackendBlock.js +7 -113
  9. package/dist/format/carbonEffect/carbonEffectBackendBlock.js.map +1 -1
  10. package/dist/format/carbonEffect/carbonEffectRecords.js +258 -66
  11. package/dist/format/carbonEffect/carbonEffectRecords.js.map +1 -1
  12. package/dist/format/carbonEffect/carbonEffectResourceTransform.js +197 -0
  13. package/dist/format/carbonEffect/carbonEffectResourceTransform.js.map +1 -0
  14. package/dist/format/effect/effectBodyInventory.js +146 -0
  15. package/dist/format/effect/effectBodyInventory.js.map +1 -0
  16. package/dist/format/effect/effectPermutationGraph.js +13 -13
  17. package/dist/format/effect/effectPermutationGraph.js.map +1 -1
  18. package/dist/format/index.js +4 -4
  19. package/dist/formats/black/CjsBlackFormat.js +19 -1
  20. package/dist/formats/black/CjsBlackFormat.js.map +1 -1
  21. package/dist/formats/bnk/CjsBnkFormat.js +28 -9
  22. package/dist/formats/bnk/CjsBnkFormat.js.map +1 -1
  23. package/dist/formats/bnk/core/busNodes.js +252 -0
  24. package/dist/formats/bnk/core/busNodes.js.map +1 -0
  25. package/dist/formats/bnk/core/effectNodes.js +147 -0
  26. package/dist/formats/bnk/core/effectNodes.js.map +1 -0
  27. package/dist/formats/bnk/core/eventAction.js +135 -24
  28. package/dist/formats/bnk/core/eventAction.js.map +1 -1
  29. package/dist/formats/bnk/core/globalSettings.js +215 -0
  30. package/dist/formats/bnk/core/globalSettings.js.map +1 -0
  31. package/dist/formats/bnk/core/helpers.js +37 -5
  32. package/dist/formats/bnk/core/helpers.js.map +1 -1
  33. package/dist/formats/bnk/core/musicNodes.js +43 -19
  34. package/dist/formats/bnk/core/musicNodes.js.map +1 -1
  35. package/dist/formats/bnk/core/nodeBase.js +28 -7
  36. package/dist/formats/bnk/core/nodeBase.js.map +1 -1
  37. package/dist/formats/dds/core/helpers.js +40 -1
  38. package/dist/formats/dds/core/helpers.js.map +1 -1
  39. package/dist/formats/dxbc/CjsDxbcFormat.js +45 -0
  40. package/dist/formats/dxbc/CjsDxbcFormat.js.map +1 -1
  41. package/dist/formats/dxbc/core/disassemble.js +240 -0
  42. package/dist/formats/dxbc/core/disassemble.js.map +1 -0
  43. package/dist/formats/gltf/CjsGltfFormat.js +1 -39
  44. package/dist/formats/gltf/CjsGltfFormat.js.map +1 -1
  45. package/dist/formats/gltf/core/helpers.js +1 -21
  46. package/dist/formats/gltf/core/helpers.js.map +1 -1
  47. package/dist/formats/hlsl/CjsHlslFormat.js +0 -21
  48. package/dist/formats/hlsl/CjsHlslFormat.js.map +1 -1
  49. package/dist/formats/hlsl/core/carbonDescriptionToRuntime.js +118 -49
  50. package/dist/formats/hlsl/core/carbonDescriptionToRuntime.js.map +1 -1
  51. package/dist/formats/hlsl/core/detailMapFamily.js +128 -0
  52. package/dist/formats/hlsl/core/detailMapFamily.js.map +1 -0
  53. package/dist/formats/hlsl/core/localLightFamily.js +133 -0
  54. package/dist/formats/hlsl/core/localLightFamily.js.map +1 -0
  55. package/dist/formats/hlsl/core/tr2/resources/HlslEffectRes.js +79 -73
  56. package/dist/formats/hlsl/core/tr2/resources/HlslEffectRes.js.map +1 -1
  57. package/dist/formats/hlsl/core/tr2/shader/HlslEffectBindingManifest.js +8 -1
  58. package/dist/formats/hlsl/core/tr2/shader/HlslEffectBindingManifest.js.map +1 -1
  59. package/dist/formats/hlsl/core/tr2/shader/HlslEffectDescription.js +11 -691
  60. package/dist/formats/hlsl/core/tr2/shader/HlslEffectDescription.js.map +1 -1
  61. package/dist/formats/index.js +4 -2
  62. package/dist/formats/index.js.map +1 -1
  63. package/dist/formats/obj/CjsObjFormat.js +1 -22
  64. package/dist/formats/obj/CjsObjFormat.js.map +1 -1
  65. package/dist/formats/obj/core/helpers.js +1 -21
  66. package/dist/formats/obj/core/helpers.js.map +1 -1
  67. package/dist/formats/pickle/CjsPickleFormat.js +214 -0
  68. package/dist/formats/pickle/CjsPickleFormat.js.map +1 -0
  69. package/dist/formats/pickle/core/CjsPickleProtocol0Reader.js +551 -0
  70. package/dist/formats/pickle/core/CjsPickleProtocol0Reader.js.map +1 -0
  71. package/dist/formats/pickle/index.js +2 -0
  72. package/dist/formats/pickle/index.js.map +1 -0
  73. package/dist/formats/red/core/redGraph.js +11 -8
  74. package/dist/formats/red/core/redGraph.js.map +1 -1
  75. package/dist/formats/stl/CjsStlFormat.js +1 -46
  76. package/dist/formats/stl/CjsStlFormat.js.map +1 -1
  77. package/dist/formats/stl/core/helpers.js +1 -21
  78. package/dist/formats/stl/core/helpers.js.map +1 -1
  79. package/dist/formats/webgl/CjsWebglFormat.js +33 -55
  80. package/dist/formats/webgl/CjsWebglFormat.js.map +1 -1
  81. package/dist/formats/webgl/core/buildGlslEffectContainer.js +70 -0
  82. package/dist/formats/webgl/core/buildGlslEffectContainer.js.map +1 -0
  83. package/dist/formats/webgl/core/effectPackage.js +216 -108
  84. package/dist/formats/webgl/core/effectPackage.js.map +1 -1
  85. package/dist/formats/webgl/core/errors.js +1 -1
  86. package/dist/formats/webgl/core/errors.js.map +1 -1
  87. package/dist/formats/webgl/core/glsl/DxbcGlslEmitter.js +166 -24
  88. package/dist/formats/webgl/core/glsl/DxbcGlslEmitter.js.map +1 -1
  89. package/dist/formats/webgl/core/glsl/DxbcGlslOperandFormatter.js +63 -5
  90. package/dist/formats/webgl/core/glsl/DxbcGlslOperandFormatter.js.map +1 -1
  91. package/dist/formats/webgl/core/glsl/packedLightFixups.js +98 -0
  92. package/dist/formats/webgl/core/glsl/packedLightFixups.js.map +1 -0
  93. package/dist/formats/webgl/core/glslBackendBlock.js +552 -0
  94. package/dist/formats/webgl/core/glslBackendBlock.js.map +1 -0
  95. package/dist/formats/webgl/core/glslBackendBodySet.js +243 -0
  96. package/dist/formats/webgl/core/glslBackendBodySet.js.map +1 -0
  97. package/dist/formats/webgl/core/{cewgCompleteness.js → glslEffectCompleteness.js} +12 -8
  98. package/dist/formats/webgl/core/glslEffectCompleteness.js.map +1 -0
  99. package/dist/formats/webgl/core/helpers.js +33 -172
  100. package/dist/formats/webgl/core/helpers.js.map +1 -1
  101. package/dist/formats/webgl/core/inspectGlslEffectContainer.js +122 -0
  102. package/dist/formats/webgl/core/inspectGlslEffectContainer.js.map +1 -0
  103. package/dist/formats/webgl/core/readGlslEffectContainer.js +249 -0
  104. package/dist/formats/webgl/core/readGlslEffectContainer.js.map +1 -0
  105. package/dist/formats/webgpu/CjsWebgpuFormat.js +15 -15
  106. package/dist/formats/webgpu/CjsWebgpuFormat.js.map +1 -1
  107. package/dist/formats/webgpu/core/buildCarbonEffectContainer.js +37 -145
  108. package/dist/formats/webgpu/core/buildCarbonEffectContainer.js.map +1 -1
  109. package/dist/formats/webgpu/core/{cewgpu/CewgpuContainer.js → carbonWebgpu/CarbonWebgpuContainer.js} +13 -27
  110. package/dist/formats/webgpu/core/carbonWebgpu/CarbonWebgpuContainer.js.map +1 -0
  111. package/dist/formats/webgpu/core/{cewgpu → carbonWebgpu}/containerViews.js +1 -1
  112. package/dist/formats/webgpu/core/carbonWebgpu/containerViews.js.map +1 -0
  113. package/dist/formats/webgpu/core/{cewgpu → carbonWebgpu}/validateContainer.js +4 -4
  114. package/dist/formats/webgpu/core/carbonWebgpu/validateContainer.js.map +1 -0
  115. package/dist/formats/webgpu/core/effectBackendBodySet.js +18 -18
  116. package/dist/formats/webgpu/core/effectBackendBodySet.js.map +1 -1
  117. package/dist/formats/webgpu/core/errors.js +1 -1
  118. package/dist/formats/webgpu/core/errors.js.map +1 -1
  119. package/dist/formats/webgpu/core/helpers.js +22 -22
  120. package/dist/formats/webgpu/core/helpers.js.map +1 -1
  121. package/dist/formats/webgpu/core/packageEffect.js +11 -15
  122. package/dist/formats/webgpu/core/packageEffect.js.map +1 -1
  123. package/dist/formats/webgpu/core/packageEffectSelection.js +1 -1
  124. package/dist/formats/webgpu/core/packageEffectSelection.js.map +1 -1
  125. package/dist/formats/webgpu/core/wgsl/buildResourceTransformPlan.js +23 -23
  126. package/dist/formats/webgpu/core/wgsl/buildResourceTransformPlan.js.map +1 -1
  127. package/dist/formats/webgpu/core/wgsl/buildWgslSet.js +6 -6
  128. package/dist/formats/webgpu/core/wgsl/buildWgslSet.js.map +1 -1
  129. package/dist/formats/yaml/CjsYamlFormat.js +3 -3
  130. package/dist/formats/yaml/CjsYamlFormat.js.map +1 -1
  131. package/dist/formats/yaml/core/CjsYamlReader.js +4 -6
  132. package/dist/formats/yaml/core/CjsYamlReader.js.map +1 -1
  133. package/dist/formats/yaml/core/helpers.js +27 -1
  134. package/dist/formats/yaml/core/helpers.js.map +1 -1
  135. package/dist/index.js +2 -0
  136. package/dist/index.js.map +1 -1
  137. package/dist/resource/CjsLoadingObject.js +19 -0
  138. package/dist/resource/CjsLoadingObject.js.map +1 -0
  139. package/dist/resource/CjsResource.js +4 -0
  140. package/dist/resource/CjsResource.js.map +1 -1
  141. package/dist/resource/ResourceHandlerMode.js +14 -0
  142. package/dist/resource/ResourceHandlerMode.js.map +1 -0
  143. package/dist/resource/index.js +2 -0
  144. package/dist/resource/index.js.map +1 -1
  145. package/dist/resource/shader/Tr2EffectRes.js +118 -213
  146. package/dist/resource/shader/Tr2EffectRes.js.map +1 -1
  147. package/dist/resource/shader/Tr2Shader.js +37 -15
  148. package/dist/resource/shader/Tr2Shader.js.map +1 -1
  149. package/dist/resource/shader/reflection/Tr2EffectConstant.js +37 -29
  150. package/dist/resource/shader/reflection/Tr2EffectConstant.js.map +1 -1
  151. package/dist/resource/shader/reflection/Tr2EffectDescription.js +39 -34
  152. package/dist/resource/shader/reflection/Tr2EffectDescription.js.map +1 -1
  153. package/dist/resource/shader/reflection/Tr2EffectLibrary.js +51 -30
  154. package/dist/resource/shader/reflection/Tr2EffectLibrary.js.map +1 -1
  155. package/dist/resource/shader/reflection/Tr2EffectParameterAnnotation.js +45 -25
  156. package/dist/resource/shader/reflection/Tr2EffectParameterAnnotation.js.map +1 -1
  157. package/dist/resource/shader/reflection/Tr2EffectResource.js +51 -17
  158. package/dist/resource/shader/reflection/Tr2EffectResource.js.map +1 -1
  159. package/dist/resource/shader/reflection/Tr2EffectStageInput.js +219 -88
  160. package/dist/resource/shader/reflection/Tr2EffectStageInput.js.map +1 -1
  161. package/dist/resource/shader/reflection/Tr2EffectTechnique.js +28 -18
  162. package/dist/resource/shader/reflection/Tr2EffectTechnique.js.map +1 -1
  163. package/dist/resource/shader/reflection/Tr2Pass.js +96 -39
  164. package/dist/resource/shader/reflection/Tr2Pass.js.map +1 -1
  165. package/dist/resource/shader/reflection/carbonRecordFields.js +159 -0
  166. package/dist/resource/shader/reflection/carbonRecordFields.js.map +1 -0
  167. package/dist/resource/shader/reflection/shaderStage.js +22 -0
  168. package/dist/resource/shader/reflection/shaderStage.js.map +1 -0
  169. package/dist/resource/shader/sampler/Tr2SamplerSetup.js +68 -10
  170. package/dist/resource/shader/sampler/Tr2SamplerSetup.js.map +1 -1
  171. package/dist/worker/CjsResManMainThreadLoader.js +9 -7
  172. package/dist/worker/CjsResManMainThreadLoader.js.map +1 -1
  173. package/dist/worker/CjsResManWorker.js +5 -4
  174. package/dist/worker/CjsResManWorker.js.map +1 -1
  175. package/dist/worker/CjsResManWorkerLoader.js +9 -6
  176. package/dist/worker/CjsResManWorkerLoader.js.map +1 -1
  177. package/docs/README.md +2 -2
  178. package/docs/architecture.md +12 -3
  179. package/docs/concepts/resource-lifecycle.md +9 -8
  180. package/docs/concepts/shader-resource-model.md +14 -17
  181. package/docs/concepts/writing-an-engine-adapter.md +1 -1
  182. package/docs/formats/README.md +4 -2
  183. package/docs/formats/carbon-effect-container.md +127 -26
  184. package/docs/formats/dxbc/architecture.md +1 -1
  185. package/docs/formats/dxbc/reference/api.md +28 -0
  186. package/docs/formats/hlsl/README.md +3 -3
  187. package/docs/formats/hlsl/architecture.md +8 -9
  188. package/docs/formats/hlsl/guides/hydrating-json-output.md +3 -5
  189. package/docs/formats/hlsl/guides/reading-effects.md +5 -1
  190. package/docs/formats/hlsl/reference/advanced-analysis.md +20 -25
  191. package/docs/formats/hlsl/reference/api.md +0 -7
  192. package/docs/formats/hlsl/reference/json-graph.md +1 -4
  193. package/docs/formats/pickle.md +82 -0
  194. package/docs/formats/webgl/README.md +71 -13
  195. package/docs/formats/webgl/architecture.md +16 -17
  196. package/docs/formats/webgl/carbon-constant-layouts.md +3 -3
  197. package/docs/formats/webgl/decl-io.md +3 -3
  198. package/docs/formats/webgl/memory-structured.md +27 -8
  199. package/docs/formats/webgl/texture-sample.md +1 -1
  200. package/docs/formats/webgpu/README.md +3 -3
  201. package/docs/formats/webgpu/architecture.md +5 -6
  202. package/docs/formats/webgpu/formats/{cewgpu.md → carbon-webgpu.md} +10 -11
  203. package/docs/formats/webgpu/guides/effect-packaging.md +6 -8
  204. package/docs/formats/webgpu/reference/api.md +11 -12
  205. package/docs/formats/webgpu/reference/wgsl-compatibility.md +11 -8
  206. package/docs/formats/wwise.md +63 -2
  207. package/docs/reference/classes/core.md +10 -0
  208. package/docs/reference/classes/formats.md +40 -48
  209. package/docs/reference/classes/resources.md +2 -2
  210. package/docs/reference/queues.md +88 -4
  211. package/docs/reference/workers.md +7 -0
  212. package/package.json +61 -61
  213. package/dist/format/carbonEffect/carbonDescriptionFromPortable.js +0 -372
  214. package/dist/format/carbonEffect/carbonDescriptionFromPortable.js.map +0 -1
  215. package/dist/format/effect/effectReflectionPackage.js +0 -636
  216. package/dist/format/effect/effectReflectionPackage.js.map +0 -1
  217. package/dist/formats/hlsl/core/HlslReader.js +0 -16
  218. package/dist/formats/hlsl/core/HlslReader.js.map +0 -1
  219. package/dist/formats/hlsl/core/portableReflection.js +0 -848
  220. package/dist/formats/hlsl/core/portableReflection.js.map +0 -1
  221. package/dist/formats/hlsl/portable.js +0 -2
  222. package/dist/formats/hlsl/portable.js.map +0 -1
  223. package/dist/formats/webgl/core/cewg/CewgPackage.js +0 -340
  224. package/dist/formats/webgl/core/cewg/CewgPackage.js.map +0 -1
  225. package/dist/formats/webgl/core/cewg/CewgPackageBuilder.js +0 -104
  226. package/dist/formats/webgl/core/cewg/CewgPackageBuilder.js.map +0 -1
  227. package/dist/formats/webgl/core/cewg/binary.js +0 -19
  228. package/dist/formats/webgl/core/cewg/binary.js.map +0 -1
  229. package/dist/formats/webgl/core/cewgCompleteness.js.map +0 -1
  230. package/dist/formats/webgl/core/effectPackageValidation.js +0 -331
  231. package/dist/formats/webgl/core/effectPackageValidation.js.map +0 -1
  232. package/dist/formats/webgpu/core/cewgpu/CewgpuContainer.js.map +0 -1
  233. package/dist/formats/webgpu/core/cewgpu/containerViews.js.map +0 -1
  234. package/dist/formats/webgpu/core/cewgpu/validateContainer.js.map +0 -1
  235. package/dist/resource/shader/portable.js +0 -33
  236. package/dist/resource/shader/portable.js.map +0 -1
  237. package/docs/formats/hlsl/reference/portable-reflection.md +0 -141
  238. package/docs/formats/webgl/effect-reflection.md +0 -127
@@ -3,7 +3,7 @@
3
3
  Status: Stable
4
4
  Visibility: Public
5
5
  Scope: `@carbonenginejs/runtime-resource` — `src/format/carbonEffect/`, `src/format/CjsByteReader.js`, `src/format/CjsByteWriter.js`, `src/format/CjsStringTable.js`
6
- Audience: Anyone reading or writing compiled shader effect bytes, or extending `.cewgpu`
6
+ Audience: Anyone reading or writing compiled shader effect bytes, or extending `.carbonwebgpu`
7
7
  Summary: The v15 binary layout of Carbon's compiled effect files, the shared byte reader and writer that implement it, and the arena offset policy a byte-exact re-emit depends on.
8
8
 
9
9
  ## What this is
@@ -14,8 +14,10 @@ a deduplicated blob arena ("string table"), one dense offset-table row per
14
14
  permutation, and one stored description blob per distinct encoded body.
15
15
 
16
16
  This package implements it as a shared reader and writer, verified byte-exact against
17
- CCP's own shipped files. It is the wire foundation for `.cewgpu`. The `.cewg`
18
- WebGL package retains its separate `CEWG` magic and chunk layout.
17
+ CCP's own shipped files. It is the wire foundation for **both** browser backends:
18
+ `.carbonwebgpu` and `.carbonwebgl` are the same Carbon v15 container, differing
19
+ only in the program text occupying each stage's slot and in the optional
20
+ per-pass backend block. Neither carries a private magic or chunk layout.
19
21
 
20
22
  | module | role |
21
23
  |---|---|
@@ -26,16 +28,77 @@ WebGL package retains its separate `CEWG` magic and chunk layout.
26
28
  | `src/format/carbonEffect/CjsCarbonEffectReader.js` | container reader and structural checks |
27
29
  | `src/format/carbonEffect/CjsCarbonEffectWriter.js` | container writer, offset arithmetic and alias dedupe |
28
30
 
29
- ## Why v15 only
30
-
31
- The reader and writer accept and emit version 15 and nothing else.
32
-
33
- Carbon's own reader accepts versions 2 through 15, but its v13/v14 branches mark
34
- the field-order boundaries as uncertain. Version 15 is the version with an
35
- authoritative writer to check against, and the entire audited shipped corpus at
36
- build 3444265 is v15 3222 files across `effect.dx11` and `effect.dx12`, plus
37
- the same 537 shaders again under `effect.metal`. Nothing older appears in that
38
- audited corpus.
31
+ ## Versioning: how Carbon does it, and what this reader does instead
32
+
33
+ ### Carbon validates the version, then threads it through parsing
34
+
35
+ Carbon reads the version dword once, in `Tr2EffectRes`, and rejects anything
36
+ outside 2..15 (`Tr2EffectRes.cpp:209`). Inside that range it never branches to
37
+ accept or refuse again the version becomes a *parsing input*. It is first used
38
+ by the container reader itself, then **passed as a parameter** into the effect
39
+ description's read call (`Tr2EffectRes.cpp:128`), alongside the buffer, its
40
+ size, and the string table.
41
+
42
+ From there it is threaded through every record reader in
43
+ `Tr2EffectDescription.cpp` — `ReadConstant`, `ReadResource`,
44
+ `ReadPipelineInputs`, `ReadRegisters`, `ReadInput`, and the pass/technique loop
45
+ — which branch on it at 22 conditional sites using 16 distinct comparisons:
46
+
47
+ | Threshold | What it decides |
48
+ | --- | --- |
49
+ | `> 8` | Whether a stage carries a register signature **at all** |
50
+ | `< 14` | Where pipeline inputs are read, and whether registers are read beside them |
51
+ | `< 11` | How a constant's type byte is decoded — older files remap a legacy enum, so the byte count is unchanged but its meaning is not |
52
+ | `>= 13` | Whether a resource carries `arrayElements` at all; below 13 it is absent and defaults to 1 |
53
+ | `< 5`, `< 4`, `>= 3`, `>= 4` | Stage-input presence and ordering |
54
+ | `> 6`, `> 9`, `> 10`, `> 12`, `>= 8` | Per-record field presence |
55
+ | `> 13`, `>= 14` | Library and pipeline-input placement |
56
+
57
+ The consequence worth internalising: **the version determines the byte layout at
58
+ nearly every level of the reflection graph**, not just the header. A v8 file and
59
+ a v15 file are not the same records with a different preamble.
60
+
61
+ Note where the version does *not* reach: `Tr2Shader` itself has no version field
62
+ and no version-dependent branch. Container/header parsing and description
63
+ parsing between them absorb every version difference before the graph reaches
64
+ `Tr2Shader` — the container reader has its own branches (`Tr2EffectRes.cpp`
65
+ handles `< 5` for the legacy header size, `>= 15` for the compiler version and
66
+ source hash, and `> 5` for the permutation type), and the description reader
67
+ handles the rest. What emerges carries no version, so consumers above that
68
+ boundary are version-free. A port that preserves the same separation stays
69
+ consistent with Carbon.
70
+
71
+ ### What this reader does: constant-folded to 15
72
+
73
+ `readEffectDescription` takes no version parameter. This reader is
74
+ constant-folded to version 15 and rejects everything else outright.
75
+
76
+ That is a deliberate narrowing, and it is the missing piece if an older version
77
+ is ever needed. Supporting v8 does **not** mean writing a second reader; it
78
+ means restoring the parameter Carbon always had and implementing the branches
79
+ above. The writer is unaffected — reading an old version does not imply emitting
80
+ one, and this package emits v15 only.
81
+
82
+ Rejecting is the correct failure mode rather than a limitation to route around.
83
+ Applying v15 rules to an older layout can misalign fields — the comparisons
84
+ above change which fields are present and how their bytes are interpreted — and
85
+ must not be attempted. How such a read would fail in practice is not documented
86
+ here, because no legacy container has been examined to establish it.
87
+
88
+ The reader rejects with the version it read, and does not classify it further.
89
+ A leading dword of 2..14 is a *recognized legacy version number*, not evidence
90
+ that the bytes are an effect container: arbitrary data can begin with a small
91
+ integer, and the version dword alone carries no magic or checksum to confirm it.
92
+ Anything that reaches this rejection needs identifying by other means before it
93
+ is treated as a porting question rather than a wrong file.
94
+
95
+ ### Why 15 is the version that got implemented
96
+
97
+ Version 15 is the one with an authoritative writer to check against, and the
98
+ entire audited shipped corpus at build 3444265 is v15 — 3222 files across
99
+ `effect.dx11` and `effect.dx12`, plus the same 537 shaders again under
100
+ `effect.metal`. Nothing older appears in that audited corpus. Carbon's own
101
+ v13/v14 branches additionally mark their field-order boundaries as uncertain.
39
102
 
40
103
  The v15 body is byte-identical to v14. Version 15 differs from 14 only by the 36
41
104
  extra header bytes: the compiler version and the source hash.
@@ -61,10 +124,7 @@ compares only the first three. A shipped v15 header reads `01 02 06 00` —
61
124
  compiler 1.2.6.0, matching the ShaderCompiler project version. As a `u32` those
62
125
  bytes are `0x00060201`, which means nothing. `HlslEffectRes` historically read
63
126
  the field as a dword; it now also exposes `m_compilerVersionBytes`, which is the
64
- truthful reading and what new code should use. The dword form survives only
65
- because it is republished as `source.compilerVersion` in the portable
66
- reflection, where it is asserted to be an unsigned integer and covered by a
67
- package digest.
127
+ truthful reading and what new code should use.
68
128
 
69
129
  The compiler's rebuild check reads the 32-byte hash, while the runtime skips it.
70
130
  It is provenance, not integrity.
@@ -318,6 +378,19 @@ surviving twin. The row is **kept**, so the offset table stays dense while the
318
378
  file stores each distinct body once. Across the shipped corpus 22% of files
319
379
  alias, at roughly 2.1 rows per distinct body.
320
380
 
381
+ `enumerateUniqueEffectBodies` recovers that grouping without decoding a single
382
+ body. It inspects source records and bytes only, and returns
383
+ first-occurrence-ordered groups, each holding one canonical
384
+ `permutationIndex`/`sourceRecord` plus every byte-identical alias. Exact range
385
+ aliases are the fast path; distinct ranges are fingerprinted and then compared
386
+ byte for byte. It caps the Cartesian body table at 65,536 records
387
+ (`EFFECT_BODY_COUNT_MAX`) and rejects partial overlaps, because a partial
388
+ overlap means two bodies claim the same bytes. It decodes nothing: it reads
389
+ `m_offsets` and slices `m_data` directly, so a backend packager can inventory an
390
+ effect without disturbing anything a later read depends on. It is internal
391
+ (`src/format/effect/`), not a published export; the backend body set is its one
392
+ consumer.
393
+
321
394
  ## Offset-table density
322
395
 
323
396
  Carbon indexes the offset table **positionally** and never reads each row's
@@ -360,15 +433,16 @@ close.
360
433
  layout. Current packaging emits no such prefix and the former helper is no
361
434
  longer part of the package.
362
435
 
363
- **Current WebGPU wire.** CEWGPU bytes have no envelope, magic, `payloadKind`,
436
+ **Current WebGPU wire.** Carbon WebGPU bytes have no envelope, magic, `payloadKind`,
364
437
  or independent container version. They are bare Carbon v15 records resolved
365
- from `effect.webgpu/`, with one optional per-pass backend block. `.cewg`
366
- remains a separate CEWG chunk format rather than this Carbon-record wire.
438
+ from `effect.webgpu/`, with one optional per-pass backend block. `.carbonwebgl`
439
+ remains a separate Carbon WebGL chunk format rather than this Carbon-record wire.
367
440
 
368
- `CewgpuContainer` reads the optional blocks, and the shared record reader can
369
- auto-detect them from a description's declared size. Direct conversion from
370
- those descriptions into the portable reflection envelope consumed by
371
- `Tr2EffectRes` remains an adapter boundary.
441
+ `CarbonWebgpuContainer` reads the optional blocks, and the shared record reader can
442
+ auto-detect them from a description's declared size. There is no adapter
443
+ boundary left between them and the runtime: `Tr2EffectRes.DoLoad` retains a
444
+ `CjsCarbonEffectReader` over the same bytes and `Tr2Shader.fromCarbonBinary`
445
+ builds the device-free graph from one description record.
372
446
 
373
447
  Versioning remains local to what it versions: Carbon's version dword governs
374
448
  the Carbon region, while `blobVersion` governs the optional backend block. An
@@ -394,8 +468,23 @@ disjointness.
394
468
 
395
469
  **Env-gated real-file proof.** `test/format/carbon-effect-corpus.test.js`, enabled
396
470
  with `CARBON_EFFECT_CORPUS_DIR`. Game bytes are never committed. Supply a
397
- separately acquired corpus at pinned build 3444265. The test re-emits each file
398
- three ways:
471
+ separately acquired corpus at pinned build 3444265.
472
+
473
+ `CARBON_EFFECT_CORPUS_DIR` must point at a **materialised tree of source
474
+ effects** — real `.sm_hi` / `.sm_lo` / `.sm_depth` filenames under
475
+ `effect.dx11`, `effect.dx12`, or `effect.gles2`, which is what the walker
476
+ filters on. Two nearby directories look like corpora and are not:
477
+
478
+ - a content-addressed resource store (hash-named files, no extensions) matches
479
+ nothing and fails as `no compiled effect files found` — the walker is
480
+ extension-driven, so an unextracted cache silently yields zero files;
481
+ - a directory of **our own translated output** keeps the `.sm_*` names but
482
+ holds `Carbon WebGL`/`Carbon WebGPU` containers, and fails with
483
+ `Unsupported Carbon effect version 1196901699` — that number is the ASCII
484
+ `Carbon WebGL` magic read as a version dword.
485
+
486
+ Both failures are loud, which is the intended behaviour; neither is a defect.
487
+ The test re-emits each file three ways:
399
488
 
400
489
  1. every description blob through the file's own arena — proves the field order;
401
490
  2. the whole container from raw bodies and the source arena — proves the header
@@ -417,6 +506,18 @@ arena blob, so the sorted-offset policy reproduces CCP's arena exactly.
417
506
  Supply a separately acquired corpus and set `CARBON_EFFECT_CORPUS_DIR`; no
418
507
  corpus data ships with the package.
419
508
 
509
+ **The same proof, one level up.** `test/resource/effect-res-corpus.test.js`
510
+ closes the loop through the resource classes rather than the records: bytes →
511
+ `Tr2EffectRes.DoLoad` → `Tr2Shader.fromCarbonBinary` → the device-free graph →
512
+ `toCarbonBinary()` → records → bytes. Running the record-level round trip on the
513
+ same files with no classes involved is the negative control; when the two
514
+ disagree, the difference is the class layer's, and that is how the three
515
+ fidelity losses it once had were found and closed — non-dynamic sampler names
516
+ (1,631 files), authored pass-stage order (21 files), and the offset word of a
517
+ zero-size blob (150 files), which is not always the `0xffffffff` sentinel.
518
+ Carbon's *runtime* discards the first two; the file does not, so re-emitting the
519
+ file it came from means keeping them.
520
+
420
521
  That result is the container port's central evidence. The same reader and the same
421
522
  writer reproduce, byte for byte, files whose program payloads are DXBC in two
422
523
  dialects and AIR — with no language field anywhere in the format. The metadata
@@ -46,7 +46,7 @@ them.
46
46
  belong to `format-hlsl`.
47
47
  - GLSL ES generation and WebGL-specific register/storage policy belong to
48
48
  `format-webgl`.
49
- - WGSL generation, pass-global WebGPU binding allocation, and CEWGPU assembly
49
+ - WGSL generation, pass-global WebGPU binding allocation, and Carbon WebGPU assembly
50
50
  belong to `format-webgpu`.
51
51
  - GPU shader-module and pipeline realization belongs to engine packages.
52
52
 
@@ -45,6 +45,7 @@ const summary = reader.Inspect(shaderBytes);
45
45
  | `isDxbc(bytes)` | Returns `true` when input starts with the DXBC magic; invalid input returns `false`. |
46
46
  | `read(bytes, options?)` | Reads one DXBC payload. |
47
47
  | `inspect(bytes, options?)` | Inspects one payload without instruction decoding. |
48
+ | `disassemble(input, options?)` | Returns an assembly listing of the decoded instruction stream. |
48
49
  | `toJSON(value)` | Deep-converts supported values to JSON-compatible data. |
49
50
 
50
51
  The class also exposes `OUTPUT_JSON`, `OUTPUT_RAW`, input/output media metadata,
@@ -61,6 +62,33 @@ and the format's supported input type.
61
62
  Inputs may be `Uint8Array`, `ArrayBuffer`, Node `Buffer`, `DataView`, or another
62
63
  array-buffer view.
63
64
 
65
+ ## Disassembly
66
+
67
+ `disassemble` renders decoded instructions as assembly text. It exists so a
68
+ translated shader can be checked against the bytecode it was translated from:
69
+ comparing emitted GLSL or WGSL against the original HLSL proves nothing, because
70
+ the compiler that produced the bytecode already rewrote the program, and an
71
+ emitter cannot be its own oracle.
72
+
73
+ ```js
74
+ const listing = CjsDxbcFormat.disassemble(stageBytes);
75
+ ```
76
+
77
+ Bytes, a previous `read` result, and a decoder record are all accepted, so a
78
+ caller that already decoded the payload does not decode it twice.
79
+
80
+ | Option | Default | Meaning |
81
+ | --- | --- | --- |
82
+ | `declarations` | `true` | Include declaration instructions. |
83
+ | `numbers` | `true` | Prefix executable instructions with their index. Declarations are not numbered, so the numbering matches the order a backend walks. |
84
+ | `indent` | `true` | Indent control-flow bodies. |
85
+
86
+ Component selection prints exactly as encoded rather than normalized. A
87
+ destination mask selects source components by position, so `mad r[2].xyw,
88
+ r[2].xxxx, v[3].xyxz, r[5].xyxz` writes `x`, `y`, and `w` from source components
89
+ 0, 1, and 3. Normalizing that away would hide the most common class of
90
+ translation mistake.
91
+
64
92
  ## Errors
65
93
 
66
94
  Invalid options and unsupported input types throw `TypeError`. Malformed DXBC
@@ -27,8 +27,9 @@ const metadata = CjsHlslFormat.read(bytes, {
27
27
  shader bytecode.
28
28
  - Translation backends such as `@carbonenginejs/runtime-resource/formats/webgpu` can consume
29
29
  the metadata and bytecode through their own integration layers.
30
- - `@carbonenginejs/runtime-resource` consumes packaged portable reflection and
31
- owns canonical `Tr2EffectRes`/`Tr2Shader` hydration, selection, and caching.
30
+ - `@carbonenginejs/runtime-resource` reads the compiled container directly and
31
+ owns canonical `Tr2EffectRes`/`Tr2Shader` construction, selection, and
32
+ caching.
32
33
  - `@carbonenginejs/runtime-trinity` consumes that shader graph through its
33
34
  mutable effect/material facade, parameters, options, and sampler overrides.
34
35
 
@@ -43,7 +44,6 @@ construct canonical runtime model instances, or provide a rendering runtime.
43
44
  - [API reference](reference/api.md)
44
45
  - [Advanced analysis exports](reference/advanced-analysis.md)
45
46
  - [JSON graph reference](reference/json-graph.md)
46
- - [Portable body reflection](reference/portable-reflection.md)
47
47
  - [Class catalog](reference/classes/README.md)
48
48
 
49
49
  ## Compatibility
@@ -27,8 +27,9 @@ caller bytes
27
27
  -> json | metadata | raw output
28
28
  ```
29
29
 
30
- `src/CjsHlslFormat.js` is the supported reader boundary. Binary utilities and
31
- format-shaped compatibility models live below `src/core` and `src/carbon`.
30
+ `src/formats/hlsl/CjsHlslFormat.js` is the supported reader boundary. Binary
31
+ utilities and format-shaped compatibility models live below
32
+ `src/formats/hlsl/core` and `src/formats/hlsl/core/tr2`.
32
33
  Their same-named `Tr2*` classes are internal parser DTOs, not canonical runtime
33
34
  model identity, and are not independent package entry points.
34
35
 
@@ -40,12 +41,9 @@ model identity, and are not independent package entry points.
40
41
  planning.
41
42
  - `raw` exposes internal effect-model instances for advanced tooling and is
42
43
  not a stable schema.
43
- - `@carbonenginejs/runtime-resource/formats/hlsl/portable` copies one exact body into a versioned,
44
- runtime-neutral source-reflection contract. Authored defaults and programs stay
45
- separate from mutable renderer realization.
46
44
 
47
- `runtime-resource` owns canonical `Tr2EffectRes`/`Tr2Shader` hydration,
48
- permutation selection, and caching. `runtime-trinity` owns the mutable
45
+ `runtime-resource` owns canonical `Tr2EffectRes`/`Tr2Shader` construction,
46
+ permutation selection, and caching, reading the compiled container directly. `runtime-trinity` owns the mutable
49
47
  effect/material facade, parameters, authored options, and sampler overrides.
50
48
  Engines own GPU realization.
51
49
 
@@ -55,8 +53,9 @@ The graph shapes are documented in
55
53
  ## Platform boundary
56
54
 
57
55
  `read`, `inspect`, and `toJSON` operate on caller-provided bytes and work
58
- without filesystem access. `readFile` and the CLI are Node-specific adapters.
59
- The package has no runtime dependencies.
56
+ without filesystem access; the published package is browser-only and never
57
+ imports Node modules. Node callers read bytes themselves (`node:fs`) and pass
58
+ them to `read`. The package has no runtime dependencies.
60
59
 
61
60
  ## Error boundary
62
61
 
@@ -55,8 +55,6 @@ records without a class key remain plain JSON-compatible data. Depending on
55
55
  the caller's constructors as a serialization schema is discouraged; persist
56
56
  the documented graph fields instead.
57
57
 
58
- This feature is caller-selected JSON projection only. It does not consume the
59
- versioned portable-reflection contract and does not construct canonical
60
- runtime `Tr2EffectRes` or `Tr2Shader` instances. Canonical
61
- `CJS_EFFECT_BODY_REFLECTION` hydration belongs to
62
- `@carbonenginejs/runtime-resource`.
58
+ This feature is caller-selected JSON projection only. It does not construct
59
+ canonical runtime `Tr2EffectRes` or `Tr2Shader` instances. Those are built
60
+ from the compiled container bytes by `@carbonenginejs/runtime-resource`.
@@ -53,8 +53,12 @@ render-state records needed for inspection and pipeline planning.
53
53
 
54
54
  ## Read a file in Node
55
55
 
56
+ The package itself is browser-only; Node callers supply the bytes:
57
+
56
58
  ```js
57
- const effect = await CjsHlslFormat.readFile("effect.sm_hi");
59
+ import { readFile } from "node:fs/promises";
60
+
61
+ const effect = CjsHlslFormat.read(await readFile("effect.sm_hi"), { source: "effect.sm_hi" });
58
62
  ```
59
63
 
60
64
  The CLI provides the same metadata workflow:
@@ -31,36 +31,31 @@ an effect description is available.
31
31
 
32
32
  The returned internal parser-DTO effect resource also has
33
33
  `GetShaderByIndex(index)`. It decodes one exact permutation-table slot without
34
- applying global or local option overrides; it is distinct from the canonical
35
- runtime-resource method that hydrates and caches `Tr2Shader`. Use the versioned
36
- portable subpath when the result must cross a package or serialization
37
- boundary. Its `enumerateUniqueEffectBodies(effectRes)` helper inventories
38
- first-seen raw body identities without populating the parser's mutable shader
39
- cache.
34
+ applying global or local option overrides, so a body-table index stays stable
35
+ even when an application has set global effect options. It is distinct from the
36
+ canonical `Tr2EffectRes.GetShaderByIndex`, which resolves options, hydrates a
37
+ `Tr2Shader`, and caches it per index.
40
38
 
41
- ## `Tr2EffectBindingManifest`
39
+ ## Binding manifest
42
40
 
43
- `Tr2EffectBindingManifest` is an exported advanced class that derives
44
- register-named constant, resource, sampler, and UAV bindings from an internal
45
- effect description.
41
+ `analysis.bindingManifest` derives register-named constant, resource, sampler,
42
+ and UAV bindings from an internal effect description. Its class is internal to
43
+ the format and is not a published export; treat the manifest as data.
46
44
 
47
- ```js
48
- import {
49
- readEffectAnalysis,
50
- Tr2EffectBindingManifest
51
- } from "@carbonenginejs/runtime-resource/formats/hlsl";
52
-
53
- const analysis = readEffectAnalysis(bytes);
54
- analysis.bindingManifest instanceof Tr2EffectBindingManifest;
55
- ```
45
+ ## Source truth and realization
56
46
 
57
- ## Render-context helpers
47
+ Nothing this subpath returns carries a renderer handle: no shader, program,
48
+ render-state, sampler, or library handles, and no resource sets, heap-view
49
+ arrays, backend layouts, masks, sort values, or caches. Engines own all of
50
+ that. Two consequences are easy to trip over:
58
51
 
59
- `Tr2RenderContextEnum` contains the format's stage and render-context numeric
60
- constants. `tr2ShaderStageName(value)` maps a known stage value to its
61
- readable name. Both are advanced compatibility helpers rather than a stable
62
- cross-package enumeration contract.
52
+ - Register dynamic classification is not persisted. It follows per-frame
53
+ reader and engine policy, and is not an authored binary field.
54
+ - Authored constant defaults are separate from the raw model's mutable
55
+ `constantValues`. Carbon-compatible sampler-heap realization may zero-extend
56
+ the latter; it cannot change the authored source prefix.
63
57
 
64
58
  ## Related documentation
65
59
 
66
- - [Portable body reflection](portable-reflection.md)
60
+ - [JSON and metadata graphs](json-graph.md)
61
+ - [Carbon compiled-effect container](../../carbon-effect-container.md)
@@ -18,11 +18,6 @@ import { CjsHlslFormat as NamedReader } from "@carbonenginejs/runtime-resource/f
18
18
  Additional named exports intended for advanced tooling are documented
19
19
  separately in [advanced-analysis.md](advanced-analysis.md).
20
20
 
21
- The versioned `@carbonenginejs/runtime-resource/formats/hlsl/portable` subpath exposes exact
22
- version-15 body-index reflection plus first-seen exact-byte unique-body
23
- enumeration for backend packagers. See
24
- [portable-reflection.md](portable-reflection.md).
25
-
26
21
  ## Constructor
27
22
 
28
23
  ```js
@@ -58,7 +53,6 @@ Options are reusable on an instance and can be overridden per call:
58
53
  - `CjsHlslFormat.read(input, options)` performs a one-shot read.
59
54
  - `CjsHlslFormat.inspect(input, options)` performs a one-shot inspection.
60
55
  - `CjsHlslFormat.toJSON(value)` converts output to JSON-compatible data.
61
- - `CjsHlslFormat.readFile(path, options)` reads and parses a file in Node.
62
56
 
63
57
  Inputs may be `Uint8Array`, `ArrayBuffer`, `Buffer`, or `DataView` values.
64
58
  Supported container versions are 8 through 15.
@@ -94,5 +88,4 @@ current working directory.
94
88
  - [Reading effects](../guides/reading-effects.md)
95
89
  - [Hydrating JSON output](../guides/hydrating-json-output.md)
96
90
  - [Advanced analysis exports](advanced-analysis.md)
97
- - [Portable body reflection](portable-reflection.md)
98
91
  - [Class catalog](classes/README.md)
@@ -81,9 +81,7 @@ serialization-safe schema.
81
81
 
82
82
  Raw stage and library inputs retain authored `sourceConstantValueSize` /
83
83
  `sourceConstantValues` separately from Carbon-compatible mutable constant
84
- buffers. Libraries retain their source `cjsShaderBytecode` directly. Use
85
- `@carbonenginejs/runtime-resource/formats/hlsl/portable` to copy those facts into a validated
86
- handle-free contract.
84
+ buffers. Libraries retain their source `cjsShaderBytecode` directly.
87
85
 
88
86
  ## Compatibility and failures
89
87
 
@@ -96,5 +94,4 @@ rather than being filled with guessed values.
96
94
 
97
95
  - [API reference](api.md)
98
96
  - [Advanced analysis exports](advanced-analysis.md)
99
- - [Portable body reflection](portable-reflection.md)
100
97
  - [Reading effects](../guides/reading-effects.md)
@@ -0,0 +1,82 @@
1
+ # Data-only pickle protocol 0
2
+
3
+ Status: Experimental
4
+ Scope: `@carbonenginejs/runtime-resource/formats/pickle`
5
+ Audience: Resource integrators handling inert legacy data caches
6
+ Summary: Defines safe protocol-0 pickle decoding without Python object reconstruction.
7
+
8
+ ## Boundary
9
+
10
+ `CjsPickleFormat` reads `.pickle` bytes through an internal protocol-0 reader.
11
+ It is a data decoder, not a general Python unpickler. It never imports a
12
+ module, resolves a global, calls a reducer, follows a persistent ID, or
13
+ constructs a Python object. Every executable, object-bearing, newer-protocol,
14
+ or unknown opcode fails at its exact byte offset.
15
+
16
+ The initial reader accepts the protocol-0 scalar, string, list, tuple,
17
+ dictionary, memo, append, and set-item operations required by inert data
18
+ graphs. Lists and tuples become JavaScript arrays. Integers outside the safe
19
+ JavaScript range become lossless decimal strings. Dictionary keys must be
20
+ strings or safe integers; collisions introduced by JSON key normalization are
21
+ rejected.
22
+
23
+ ## Outputs
24
+
25
+ The default `json` output returns a plain JSON-compatible graph and rejects
26
+ cycles. `payload` and debug `raw` output preserve memo aliases and may contain
27
+ cycles:
28
+
29
+ ```js
30
+ import { CjsPickleFormat } from
31
+ "@carbonenginejs/runtime-resource/formats/pickle";
32
+
33
+ const values = CjsPickleFormat.read(bytes);
34
+ const exactGraph = CjsPickleFormat.read(bytes, { emit: "payload" });
35
+ ```
36
+
37
+ `CjsPickleFormat.supportedProtocols` currently contains only protocol `0`.
38
+ The public format name follows the `.pickle` extension so future protocol
39
+ dispatch can be added without changing resource registration. Unsupported
40
+ protocols continue to fail closed until their data-only semantics are
41
+ implemented and tested.
42
+
43
+ ## Resource registration
44
+
45
+ The format contains no domain target policy. A caller registers the extension
46
+ and decides whether an exact resource path returns raw decoded data or hydrates
47
+ a target:
48
+
49
+ ```js
50
+ import {
51
+ CjsLoadingObject,
52
+ CjsResMan
53
+ } from "@carbonenginejs/runtime-resource";
54
+ import { CjsPickleFormat } from
55
+ "@carbonenginejs/runtime-resource/formats/pickle";
56
+
57
+ const resMan = new CjsResMan({ source });
58
+
59
+ resMan.RegisterExtension("pickle", CjsLoadingObject, {
60
+ Format: CjsPickleFormat,
61
+ Identify(_values, context) {
62
+ return context.resFilePath === "res:/data/profile.pickle";
63
+ }
64
+ });
65
+ ```
66
+
67
+ Resource paths, extensions, and filenames in the context are normalized to
68
+ lowercase. URL-backed sources additionally expose the exact translated URL.
69
+
70
+ ## Limits
71
+
72
+ Every read is bounded by configurable positive safe-integer limits for input
73
+ bytes, operations, stack depth, memo size and IDs, container items, and string
74
+ bytes. Defaults are conservative for ordinary resource files. Callers may
75
+ lower limits per profile or read; raising them remains an explicit trust and
76
+ memory decision.
77
+
78
+ ## Related documentation
79
+
80
+ - [Format subpaths](README.md)
81
+ - [Queues, publication, and registration](../reference/queues.md)
82
+ - [Browser worker execution](../reference/workers.md)