json-toon-parser 1.0.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +21 -0
- package/README.md +111 -0
- package/dist/cli.d.ts +3 -0
- package/dist/cli.d.ts.map +1 -0
- package/dist/cli.js +62 -0
- package/dist/cli.js.map +1 -0
- package/dist/index.d.ts +42 -0
- package/dist/index.d.ts.map +1 -0
- package/dist/index.js +112 -0
- package/dist/index.js.map +1 -0
- package/dist/types.d.ts +15 -0
- package/dist/types.d.ts.map +1 -0
- package/dist/types.js +2 -0
- package/dist/types.js.map +1 -0
- package/package.json +56 -0
package/LICENSE
ADDED
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
MIT License
|
|
2
|
+
|
|
3
|
+
Copyright (c) 2024 Bhushan Chaudhari
|
|
4
|
+
|
|
5
|
+
Permission is hereby granted, free of charge, to any person obtaining a copy
|
|
6
|
+
of this software and associated documentation files (the "Software"), to deal
|
|
7
|
+
in the Software without restriction, including without limitation the rights
|
|
8
|
+
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
|
9
|
+
copies of the Software, and to permit persons to whom the Software is
|
|
10
|
+
furnished to do so, subject to the following conditions:
|
|
11
|
+
|
|
12
|
+
The above copyright notice and this permission notice shall be included in all
|
|
13
|
+
copies or substantial portions of the Software.
|
|
14
|
+
|
|
15
|
+
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
|
16
|
+
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
|
17
|
+
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
|
18
|
+
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
|
19
|
+
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
|
20
|
+
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
|
21
|
+
SOFTWARE.
|
package/README.md
ADDED
|
@@ -0,0 +1,111 @@
|
|
|
1
|
+
# json-toon-parser
|
|
2
|
+
|
|
3
|
+
🚀 Compare and convert JSON to TOON format with token efficiency analysis.
|
|
4
|
+
|
|
5
|
+
## Installation
|
|
6
|
+
```bash
|
|
7
|
+
npm install -g json-toon-parser
|
|
8
|
+
```
|
|
9
|
+
|
|
10
|
+
## CLI Usage
|
|
11
|
+
```bash
|
|
12
|
+
# Compare JSON with TOON format
|
|
13
|
+
json-toon compare data.json
|
|
14
|
+
|
|
15
|
+
# Convert JSON to TOON
|
|
16
|
+
json-toon convert data.json output.toon
|
|
17
|
+
|
|
18
|
+
# Update JSON and generate comparison
|
|
19
|
+
json-toon update data.json
|
|
20
|
+
```
|
|
21
|
+
|
|
22
|
+
## Programmatic Usage
|
|
23
|
+
```javascript
|
|
24
|
+
import { compare, convertFile, jsonToToon } from 'json-toon-parser';
|
|
25
|
+
|
|
26
|
+
// Compare JSON data
|
|
27
|
+
const result = compare({ name: "John", age: 30 }, { saveToFile: true });
|
|
28
|
+
console.log(result);
|
|
29
|
+
// Output: { jsonTokens: 15, toonTokens: 8, savings: 7, savingsPercentage: 46.67 }
|
|
30
|
+
|
|
31
|
+
// Convert file
|
|
32
|
+
const outputPath = convertFile('input.json', 'output.toon');
|
|
33
|
+
console.log(`Saved to: ${outputPath}`);
|
|
34
|
+
|
|
35
|
+
// Convert JSON to TOON string
|
|
36
|
+
const toonString = jsonToToon({ hello: "world" });
|
|
37
|
+
```
|
|
38
|
+
|
|
39
|
+
## Features
|
|
40
|
+
|
|
41
|
+
- ✅ Convert JSON to TOON format
|
|
42
|
+
- 📊 Compare token efficiency between formats
|
|
43
|
+
- 💾 Auto-save TOON files with timestamps
|
|
44
|
+
- 🔧 CLI tool for easy usage
|
|
45
|
+
- 📦 Programmatic API for Node.js
|
|
46
|
+
- 🎯 TypeScript support with full type definitions
|
|
47
|
+
|
|
48
|
+
## API
|
|
49
|
+
|
|
50
|
+
### `compare(jsonData, options?)`
|
|
51
|
+
Compare JSON data with TOON format and calculate token efficiency.
|
|
52
|
+
|
|
53
|
+
**Parameters:**
|
|
54
|
+
- `jsonData`: Any JSON-serializable data
|
|
55
|
+
- `options`: Optional configuration
|
|
56
|
+
- `outputDir`: Output directory (default: "output")
|
|
57
|
+
- `saveToFile`: Save TOON file (default: false)
|
|
58
|
+
- `prettyJson`: Pretty print JSON (default: true)
|
|
59
|
+
- `fileName`: Custom filename
|
|
60
|
+
|
|
61
|
+
**Returns:** `ComparisonResult` with token counts and savings
|
|
62
|
+
|
|
63
|
+
### `compareFile(filePath, options?)`
|
|
64
|
+
Compare JSON file with TOON format.
|
|
65
|
+
|
|
66
|
+
### `convertFile(inputPath, outputPath?)`
|
|
67
|
+
Convert JSON file to TOON format and save.
|
|
68
|
+
|
|
69
|
+
### `jsonToToon(jsonData)`
|
|
70
|
+
Convert JSON data to TOON format string.
|
|
71
|
+
|
|
72
|
+
### `updateJsonFile(filePath, outputPath?)`
|
|
73
|
+
Update JSON file and generate TOON comparison.
|
|
74
|
+
|
|
75
|
+
### `formatResult(result)`
|
|
76
|
+
Format comparison result for display.
|
|
77
|
+
|
|
78
|
+
## Example Output
|
|
79
|
+
```
|
|
80
|
+
============================================================
|
|
81
|
+
COMPARISON RESULTS
|
|
82
|
+
============================================================
|
|
83
|
+
JSON Tokens: 150
|
|
84
|
+
TOON Tokens: 95
|
|
85
|
+
Token Savings: 55
|
|
86
|
+
Efficiency: 36.67% reduction
|
|
87
|
+
Output File: output/data_2024-01-15T10-30-45.toon
|
|
88
|
+
============================================================
|
|
89
|
+
```
|
|
90
|
+
|
|
91
|
+
## Use Cases
|
|
92
|
+
|
|
93
|
+
- 🔍 **Token Analysis**: Compare token efficiency between formats
|
|
94
|
+
- 💾 **Data Compression**: Convert JSON to more efficient TOON format
|
|
95
|
+
- 📊 **Batch Processing**: Process multiple JSON files
|
|
96
|
+
- 🛠️ **CI/CD Integration**: Automate format conversion
|
|
97
|
+
- 📈 **Cost Optimization**: Reduce API token usage for LLMs
|
|
98
|
+
|
|
99
|
+
## Requirements
|
|
100
|
+
|
|
101
|
+
- Node.js >= 18.0.0
|
|
102
|
+
|
|
103
|
+
## License
|
|
104
|
+
|
|
105
|
+
MIT © Bhushan Chaudhari
|
|
106
|
+
|
|
107
|
+
## Links
|
|
108
|
+
|
|
109
|
+
- [GitHub Repository](https://github.com/bhushan44/json-toon-parser)
|
|
110
|
+
- [Report Issues](https://github.com/bhushan44/json-toon-parser/issues)
|
|
111
|
+
- [NPM Package](https://www.npmjs.com/package/json-toon-parser)
|
package/dist/cli.d.ts
ADDED
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"cli.d.ts","sourceRoot":"","sources":["../src/cli.ts"],"names":[],"mappings":""}
|
package/dist/cli.js
ADDED
|
@@ -0,0 +1,62 @@
|
|
|
1
|
+
#!/usr/bin/env node
|
|
2
|
+
import { Command } from "commander";
|
|
3
|
+
import { compareFile, convertFile, formatResult, updateJsonFile } from "./index.js";
|
|
4
|
+
const program = new Command();
|
|
5
|
+
program
|
|
6
|
+
.name("json-toon")
|
|
7
|
+
.description("Compare and convert JSON to TOON format")
|
|
8
|
+
.version("1.0.0");
|
|
9
|
+
program
|
|
10
|
+
.command("compare")
|
|
11
|
+
.description("Compare JSON file with TOON format")
|
|
12
|
+
.argument("<file>", "JSON file to compare")
|
|
13
|
+
.option("-o, --output <dir>", "Output directory", "output")
|
|
14
|
+
.option("-s, --save", "Save TOON file", false)
|
|
15
|
+
.action((file, options) => {
|
|
16
|
+
try {
|
|
17
|
+
const result = compareFile(file, {
|
|
18
|
+
outputDir: options.output,
|
|
19
|
+
saveToFile: options.save,
|
|
20
|
+
});
|
|
21
|
+
console.log(formatResult(result));
|
|
22
|
+
}
|
|
23
|
+
catch (error) {
|
|
24
|
+
console.error("Error:", error instanceof Error ? error.message : error);
|
|
25
|
+
process.exit(1);
|
|
26
|
+
}
|
|
27
|
+
});
|
|
28
|
+
program
|
|
29
|
+
.command("convert")
|
|
30
|
+
.description("Convert JSON file to TOON format")
|
|
31
|
+
.argument("<input>", "Input JSON file")
|
|
32
|
+
.argument("[output]", "Output TOON file")
|
|
33
|
+
.action((input, output) => {
|
|
34
|
+
try {
|
|
35
|
+
const outputPath = convertFile(input, output);
|
|
36
|
+
console.log(`✅ Converted successfully!`);
|
|
37
|
+
console.log(`📁 Output: ${outputPath}`);
|
|
38
|
+
}
|
|
39
|
+
catch (error) {
|
|
40
|
+
console.error("Error:", error instanceof Error ? error.message : error);
|
|
41
|
+
process.exit(1);
|
|
42
|
+
}
|
|
43
|
+
});
|
|
44
|
+
program
|
|
45
|
+
.command("update")
|
|
46
|
+
.description("Update JSON file and generate TOON comparison")
|
|
47
|
+
.argument("<file>", "JSON file to update")
|
|
48
|
+
.option("-o, --output <file>", "Output JSON file (default: overwrites input)")
|
|
49
|
+
.action((file, options) => {
|
|
50
|
+
try {
|
|
51
|
+
const result = updateJsonFile(file, options.output);
|
|
52
|
+
console.log(`✅ JSON file updated!`);
|
|
53
|
+
console.log(formatResult(result));
|
|
54
|
+
console.log(`📁 Updated JSON: ${result.updatedJsonPath}`);
|
|
55
|
+
}
|
|
56
|
+
catch (error) {
|
|
57
|
+
console.error("Error:", error instanceof Error ? error.message : error);
|
|
58
|
+
process.exit(1);
|
|
59
|
+
}
|
|
60
|
+
});
|
|
61
|
+
program.parse();
|
|
62
|
+
//# sourceMappingURL=cli.js.map
|
package/dist/cli.js.map
ADDED
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"cli.js","sourceRoot":"","sources":["../src/cli.ts"],"names":[],"mappings":";AACA,OAAO,EAAE,OAAO,EAAE,MAAM,WAAW,CAAC;AACpC,OAAO,EAAE,WAAW,EAAE,WAAW,EAAE,YAAY,EAAE,cAAc,EAAE,MAAM,YAAY,CAAC;AAEpF,MAAM,OAAO,GAAG,IAAI,OAAO,EAAE,CAAC;AAE9B,OAAO;KACJ,IAAI,CAAC,WAAW,CAAC;KACjB,WAAW,CAAC,yCAAyC,CAAC;KACtD,OAAO,CAAC,OAAO,CAAC,CAAC;AAEpB,OAAO;KACJ,OAAO,CAAC,SAAS,CAAC;KAClB,WAAW,CAAC,oCAAoC,CAAC;KACjD,QAAQ,CAAC,QAAQ,EAAE,sBAAsB,CAAC;KAC1C,MAAM,CAAC,oBAAoB,EAAE,kBAAkB,EAAE,QAAQ,CAAC;KAC1D,MAAM,CAAC,YAAY,EAAE,gBAAgB,EAAE,KAAK,CAAC;KAC7C,MAAM,CAAC,CAAC,IAAY,EAAE,OAA0C,EAAE,EAAE;IACnE,IAAI,CAAC;QACH,MAAM,MAAM,GAAG,WAAW,CAAC,IAAI,EAAE;YAC/B,SAAS,EAAE,OAAO,CAAC,MAAM;YACzB,UAAU,EAAE,OAAO,CAAC,IAAI;SACzB,CAAC,CAAC;QACH,OAAO,CAAC,GAAG,CAAC,YAAY,CAAC,MAAM,CAAC,CAAC,CAAC;IACpC,CAAC;IAAC,OAAO,KAAK,EAAE,CAAC;QACf,OAAO,CAAC,KAAK,CAAC,QAAQ,EAAE,KAAK,YAAY,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC;QACxE,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;IAClB,CAAC;AACH,CAAC,CAAC,CAAC;AAEL,OAAO;KACJ,OAAO,CAAC,SAAS,CAAC;KAClB,WAAW,CAAC,kCAAkC,CAAC;KAC/C,QAAQ,CAAC,SAAS,EAAE,iBAAiB,CAAC;KACtC,QAAQ,CAAC,UAAU,EAAE,kBAAkB,CAAC;KACxC,MAAM,CAAC,CAAC,KAAa,EAAE,MAAe,EAAE,EAAE;IACzC,IAAI,CAAC;QACH,MAAM,UAAU,GAAG,WAAW,CAAC,KAAK,EAAE,MAAM,CAAC,CAAC;QAC9C,OAAO,CAAC,GAAG,CAAC,2BAA2B,CAAC,CAAC;QACzC,OAAO,CAAC,GAAG,CAAC,cAAc,UAAU,EAAE,CAAC,CAAC;IAC1C,CAAC;IAAC,OAAO,KAAK,EAAE,CAAC;QACf,OAAO,CAAC,KAAK,CAAC,QAAQ,EAAE,KAAK,YAAY,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC;QACxE,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;IAClB,CAAC;AACH,CAAC,CAAC,CAAC;AAEL,OAAO;KACJ,OAAO,CAAC,QAAQ,CAAC;KACjB,WAAW,CAAC,+CAA+C,CAAC;KAC5D,QAAQ,CAAC,QAAQ,EAAE,qBAAqB,CAAC;KACzC,MAAM,CAAC,qBAAqB,EAAE,8CAA8C,CAAC;KAC7E,MAAM,CAAC,CAAC,IAAY,EAAE,OAA4B,EAAE,EAAE;IACrD,IAAI,CAAC;QACH,MAAM,MAAM,GAAG,cAAc,CAAC,IAAI,EAAE,OAAO,CAAC,MAAM,CAAC,CAAC;QACpD,OAAO,CAAC,GAAG,CAAC,sBAAsB,CAAC,CAAC;QACpC,OAAO,CAAC,GAAG,CAAC,YAAY,CAAC,MAAM,CAAC,CAAC,CAAC;QAClC,OAAO,CAAC,GAAG,CAAC,oBAAoB,MAAM,CAAC,eAAe,EAAE,CAAC,CAAC;IAC5D,CAAC;IAAC,OAAO,KAAK,EAAE,CAAC;QACf,OAAO,CAAC,KAAK,CAAC,QAAQ,EAAE,KAAK,YAAY,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC;QACxE,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;IAClB,CAAC;AACH,CAAC,CAAC,CAAC;AAEL,OAAO,CAAC,KAAK,EAAE,CAAC"}
|
package/dist/index.d.ts
ADDED
|
@@ -0,0 +1,42 @@
|
|
|
1
|
+
export interface ComparisonResult {
|
|
2
|
+
jsonTokens: number;
|
|
3
|
+
toonTokens: number;
|
|
4
|
+
savings: number;
|
|
5
|
+
savingsPercentage: number;
|
|
6
|
+
toonOutput: string;
|
|
7
|
+
toonOutputPath?: string;
|
|
8
|
+
}
|
|
9
|
+
export interface CompareOptions {
|
|
10
|
+
outputDir?: string;
|
|
11
|
+
saveToFile?: boolean;
|
|
12
|
+
prettyJson?: boolean;
|
|
13
|
+
fileName?: string;
|
|
14
|
+
}
|
|
15
|
+
/**
|
|
16
|
+
* Compare JSON data with TOON format and calculate token efficiency
|
|
17
|
+
*/
|
|
18
|
+
export declare function compare(jsonData: unknown, options?: CompareOptions): ComparisonResult;
|
|
19
|
+
/**
|
|
20
|
+
* Compare JSON file with TOON format
|
|
21
|
+
*/
|
|
22
|
+
export declare function compareFile(filePath: string, options?: CompareOptions): ComparisonResult;
|
|
23
|
+
/**
|
|
24
|
+
* Convert JSON data to TOON format
|
|
25
|
+
*/
|
|
26
|
+
export declare function jsonToToon(jsonData: unknown): string;
|
|
27
|
+
/**
|
|
28
|
+
* Convert JSON file to TOON format and save
|
|
29
|
+
*/
|
|
30
|
+
export declare function convertFile(inputPath: string, outputPath?: string): string;
|
|
31
|
+
/**
|
|
32
|
+
* Update JSON file by converting it to TOON and back
|
|
33
|
+
*/
|
|
34
|
+
export declare function updateJsonFile(filePath: string, outputPath?: string): ComparisonResult & {
|
|
35
|
+
updatedJsonPath: string;
|
|
36
|
+
};
|
|
37
|
+
/**
|
|
38
|
+
* Format comparison result for display
|
|
39
|
+
*/
|
|
40
|
+
export declare function formatResult(result: ComparisonResult): string;
|
|
41
|
+
export { encode as toToon } from "@toon-format/toon";
|
|
42
|
+
//# sourceMappingURL=index.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AAKA,MAAM,WAAW,gBAAgB;IAC/B,UAAU,EAAE,MAAM,CAAC;IACnB,UAAU,EAAE,MAAM,CAAC;IACnB,OAAO,EAAE,MAAM,CAAC;IAChB,iBAAiB,EAAE,MAAM,CAAC;IAC1B,UAAU,EAAE,MAAM,CAAC;IACnB,cAAc,CAAC,EAAE,MAAM,CAAC;CACzB;AAED,MAAM,WAAW,cAAc;IAC7B,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,UAAU,CAAC,EAAE,OAAO,CAAC;IACrB,UAAU,CAAC,EAAE,OAAO,CAAC;IACrB,QAAQ,CAAC,EAAE,MAAM,CAAC;CACnB;AAED;;GAEG;AACH,wBAAgB,OAAO,CACrB,QAAQ,EAAE,OAAO,EACjB,OAAO,GAAE,cAAmB,GAC3B,gBAAgB,CAkClB;AAED;;GAEG;AACH,wBAAgB,WAAW,CACzB,QAAQ,EAAE,MAAM,EAChB,OAAO,GAAE,cAAmB,GAC3B,gBAAgB,CASlB;AAED;;GAEG;AACH,wBAAgB,UAAU,CAAC,QAAQ,EAAE,OAAO,GAAG,MAAM,CAEpD;AAED;;GAEG;AACH,wBAAgB,WAAW,CAAC,SAAS,EAAE,MAAM,EAAE,UAAU,CAAC,EAAE,MAAM,GAAG,MAAM,CAe1E;AAED;;GAEG;AACH,wBAAgB,cAAc,CAC5B,QAAQ,EAAE,MAAM,EAChB,UAAU,CAAC,EAAE,MAAM,GAClB,gBAAgB,GAAG;IAAE,eAAe,EAAE,MAAM,CAAA;CAAE,CAmBhD;AAED;;GAEG;AACH,wBAAgB,YAAY,CAAC,MAAM,EAAE,gBAAgB,GAAG,MAAM,CAiB7D;AAgBD,OAAO,EAAE,MAAM,IAAI,MAAM,EAAE,MAAM,mBAAmB,CAAC"}
|
package/dist/index.js
ADDED
|
@@ -0,0 +1,112 @@
|
|
|
1
|
+
import fs from "node:fs";
|
|
2
|
+
import path from "node:path";
|
|
3
|
+
import { encode } from "@toon-format/toon";
|
|
4
|
+
import { encode as tokenize } from "gpt-3-encoder";
|
|
5
|
+
/**
|
|
6
|
+
* Compare JSON data with TOON format and calculate token efficiency
|
|
7
|
+
*/
|
|
8
|
+
export function compare(jsonData, options = {}) {
|
|
9
|
+
const { outputDir = "output", saveToFile = false, prettyJson = true, fileName, } = options;
|
|
10
|
+
const toonOutput = encode(jsonData);
|
|
11
|
+
const jsonString = prettyJson
|
|
12
|
+
? JSON.stringify(jsonData, null, 2)
|
|
13
|
+
: JSON.stringify(jsonData);
|
|
14
|
+
const jsonTokens = tokenize(jsonString).length;
|
|
15
|
+
const toonTokens = tokenize(toonOutput).length;
|
|
16
|
+
const savings = jsonTokens - toonTokens;
|
|
17
|
+
const savingsPercentage = (savings / jsonTokens) * 100;
|
|
18
|
+
const result = {
|
|
19
|
+
jsonTokens,
|
|
20
|
+
toonTokens,
|
|
21
|
+
savings,
|
|
22
|
+
savingsPercentage,
|
|
23
|
+
toonOutput,
|
|
24
|
+
};
|
|
25
|
+
if (saveToFile) {
|
|
26
|
+
const outputPath = generateOutputPath(outputDir, fileName);
|
|
27
|
+
ensureDirectory(outputDir);
|
|
28
|
+
fs.writeFileSync(outputPath, toonOutput, "utf-8");
|
|
29
|
+
result.toonOutputPath = outputPath;
|
|
30
|
+
}
|
|
31
|
+
return result;
|
|
32
|
+
}
|
|
33
|
+
/**
|
|
34
|
+
* Compare JSON file with TOON format
|
|
35
|
+
*/
|
|
36
|
+
export function compareFile(filePath, options = {}) {
|
|
37
|
+
if (!fs.existsSync(filePath)) {
|
|
38
|
+
throw new Error(`File not found: ${filePath}`);
|
|
39
|
+
}
|
|
40
|
+
const jsonData = JSON.parse(fs.readFileSync(filePath, "utf-8"));
|
|
41
|
+
const fileName = options.fileName || path.parse(filePath).name;
|
|
42
|
+
return compare(jsonData, { ...options, fileName, saveToFile: true });
|
|
43
|
+
}
|
|
44
|
+
/**
|
|
45
|
+
* Convert JSON data to TOON format
|
|
46
|
+
*/
|
|
47
|
+
export function jsonToToon(jsonData) {
|
|
48
|
+
return encode(jsonData);
|
|
49
|
+
}
|
|
50
|
+
/**
|
|
51
|
+
* Convert JSON file to TOON format and save
|
|
52
|
+
*/
|
|
53
|
+
export function convertFile(inputPath, outputPath) {
|
|
54
|
+
if (!fs.existsSync(inputPath)) {
|
|
55
|
+
throw new Error(`File not found: ${inputPath}`);
|
|
56
|
+
}
|
|
57
|
+
const jsonData = JSON.parse(fs.readFileSync(inputPath, "utf-8"));
|
|
58
|
+
const toonOutput = encode(jsonData);
|
|
59
|
+
const finalOutputPath = outputPath || generateOutputPath("output", path.parse(inputPath).name);
|
|
60
|
+
ensureDirectory(path.dirname(finalOutputPath));
|
|
61
|
+
fs.writeFileSync(finalOutputPath, toonOutput, "utf-8");
|
|
62
|
+
return finalOutputPath;
|
|
63
|
+
}
|
|
64
|
+
/**
|
|
65
|
+
* Update JSON file by converting it to TOON and back
|
|
66
|
+
*/
|
|
67
|
+
export function updateJsonFile(filePath, outputPath) {
|
|
68
|
+
if (!fs.existsSync(filePath)) {
|
|
69
|
+
throw new Error(`File not found: ${filePath}`);
|
|
70
|
+
}
|
|
71
|
+
const originalJson = JSON.parse(fs.readFileSync(filePath, "utf-8"));
|
|
72
|
+
const result = compare(originalJson, { saveToFile: true });
|
|
73
|
+
const finalOutputPath = outputPath || filePath;
|
|
74
|
+
fs.writeFileSync(finalOutputPath, JSON.stringify(originalJson, null, 2), "utf-8");
|
|
75
|
+
return {
|
|
76
|
+
...result,
|
|
77
|
+
updatedJsonPath: finalOutputPath,
|
|
78
|
+
};
|
|
79
|
+
}
|
|
80
|
+
/**
|
|
81
|
+
* Format comparison result for display
|
|
82
|
+
*/
|
|
83
|
+
export function formatResult(result) {
|
|
84
|
+
const lines = [
|
|
85
|
+
"=".repeat(60),
|
|
86
|
+
"COMPARISON RESULTS",
|
|
87
|
+
"=".repeat(60),
|
|
88
|
+
`JSON Tokens: ${result.jsonTokens}`,
|
|
89
|
+
`TOON Tokens: ${result.toonTokens}`,
|
|
90
|
+
`Token Savings: ${result.savings}`,
|
|
91
|
+
`Efficiency: ${result.savingsPercentage.toFixed(2)}% reduction`,
|
|
92
|
+
];
|
|
93
|
+
if (result.toonOutputPath) {
|
|
94
|
+
lines.push(`Output File: ${result.toonOutputPath}`);
|
|
95
|
+
}
|
|
96
|
+
lines.push("=".repeat(60));
|
|
97
|
+
return lines.join("\n");
|
|
98
|
+
}
|
|
99
|
+
// Helper functions
|
|
100
|
+
function ensureDirectory(dir) {
|
|
101
|
+
if (!fs.existsSync(dir)) {
|
|
102
|
+
fs.mkdirSync(dir, { recursive: true });
|
|
103
|
+
}
|
|
104
|
+
}
|
|
105
|
+
function generateOutputPath(outputDir, fileName) {
|
|
106
|
+
const timestamp = new Date().toISOString().replace(/[:.]/g, "-");
|
|
107
|
+
const name = fileName || "data";
|
|
108
|
+
const finalFileName = `${name}_${timestamp}.toon`;
|
|
109
|
+
return path.join(outputDir, finalFileName);
|
|
110
|
+
}
|
|
111
|
+
export { encode as toToon } from "@toon-format/toon";
|
|
112
|
+
//# sourceMappingURL=index.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"index.js","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,MAAM,SAAS,CAAC;AACzB,OAAO,IAAI,MAAM,WAAW,CAAC;AAC7B,OAAO,EAAE,MAAM,EAAE,MAAM,mBAAmB,CAAC;AAC3C,OAAO,EAAE,MAAM,IAAI,QAAQ,EAAE,MAAM,eAAe,CAAC;AAkBnD;;GAEG;AACH,MAAM,UAAU,OAAO,CACrB,QAAiB,EACjB,UAA0B,EAAE;IAE5B,MAAM,EACJ,SAAS,GAAG,QAAQ,EACpB,UAAU,GAAG,KAAK,EAClB,UAAU,GAAG,IAAI,EACjB,QAAQ,GACT,GAAG,OAAO,CAAC;IAEZ,MAAM,UAAU,GAAG,MAAM,CAAC,QAAQ,CAAC,CAAC;IACpC,MAAM,UAAU,GAAG,UAAU;QAC3B,CAAC,CAAC,IAAI,CAAC,SAAS,CAAC,QAAQ,EAAE,IAAI,EAAE,CAAC,CAAC;QACnC,CAAC,CAAC,IAAI,CAAC,SAAS,CAAC,QAAQ,CAAC,CAAC;IAE7B,MAAM,UAAU,GAAG,QAAQ,CAAC,UAAU,CAAC,CAAC,MAAM,CAAC;IAC/C,MAAM,UAAU,GAAG,QAAQ,CAAC,UAAU,CAAC,CAAC,MAAM,CAAC;IAC/C,MAAM,OAAO,GAAG,UAAU,GAAG,UAAU,CAAC;IACxC,MAAM,iBAAiB,GAAG,CAAC,OAAO,GAAG,UAAU,CAAC,GAAG,GAAG,CAAC;IAEvD,MAAM,MAAM,GAAqB;QAC/B,UAAU;QACV,UAAU;QACV,OAAO;QACP,iBAAiB;QACjB,UAAU;KACX,CAAC;IAEF,IAAI,UAAU,EAAE,CAAC;QACf,MAAM,UAAU,GAAG,kBAAkB,CAAC,SAAS,EAAE,QAAQ,CAAC,CAAC;QAC3D,eAAe,CAAC,SAAS,CAAC,CAAC;QAC3B,EAAE,CAAC,aAAa,CAAC,UAAU,EAAE,UAAU,EAAE,OAAO,CAAC,CAAC;QAClD,MAAM,CAAC,cAAc,GAAG,UAAU,CAAC;IACrC,CAAC;IAED,OAAO,MAAM,CAAC;AAChB,CAAC;AAED;;GAEG;AACH,MAAM,UAAU,WAAW,CACzB,QAAgB,EAChB,UAA0B,EAAE;IAE5B,IAAI,CAAC,EAAE,CAAC,UAAU,CAAC,QAAQ,CAAC,EAAE,CAAC;QAC7B,MAAM,IAAI,KAAK,CAAC,mBAAmB,QAAQ,EAAE,CAAC,CAAC;IACjD,CAAC;IAED,MAAM,QAAQ,GAAG,IAAI,CAAC,KAAK,CAAC,EAAE,CAAC,YAAY,CAAC,QAAQ,EAAE,OAAO,CAAC,CAAC,CAAC;IAChE,MAAM,QAAQ,GAAG,OAAO,CAAC,QAAQ,IAAI,IAAI,CAAC,KAAK,CAAC,QAAQ,CAAC,CAAC,IAAI,CAAC;IAE/D,OAAO,OAAO,CAAC,QAAQ,EAAE,EAAE,GAAG,OAAO,EAAE,QAAQ,EAAE,UAAU,EAAE,IAAI,EAAE,CAAC,CAAC;AACvE,CAAC;AAED;;GAEG;AACH,MAAM,UAAU,UAAU,CAAC,QAAiB;IAC1C,OAAO,MAAM,CAAC,QAAQ,CAAC,CAAC;AAC1B,CAAC;AAED;;GAEG;AACH,MAAM,UAAU,WAAW,CAAC,SAAiB,EAAE,UAAmB;IAChE,IAAI,CAAC,EAAE,CAAC,UAAU,CAAC,SAAS,CAAC,EAAE,CAAC;QAC9B,MAAM,IAAI,KAAK,CAAC,mBAAmB,SAAS,EAAE,CAAC,CAAC;IAClD,CAAC;IAED,MAAM,QAAQ,GAAG,IAAI,CAAC,KAAK,CAAC,EAAE,CAAC,YAAY,CAAC,SAAS,EAAE,OAAO,CAAC,CAAC,CAAC;IACjE,MAAM,UAAU,GAAG,MAAM,CAAC,QAAQ,CAAC,CAAC;IAEpC,MAAM,eAAe,GACnB,UAAU,IAAI,kBAAkB,CAAC,QAAQ,EAAE,IAAI,CAAC,KAAK,CAAC,SAAS,CAAC,CAAC,IAAI,CAAC,CAAC;IAEzE,eAAe,CAAC,IAAI,CAAC,OAAO,CAAC,eAAe,CAAC,CAAC,CAAC;IAC/C,EAAE,CAAC,aAAa,CAAC,eAAe,EAAE,UAAU,EAAE,OAAO,CAAC,CAAC;IAEvD,OAAO,eAAe,CAAC;AACzB,CAAC;AAED;;GAEG;AACH,MAAM,UAAU,cAAc,CAC5B,QAAgB,EAChB,UAAmB;IAEnB,IAAI,CAAC,EAAE,CAAC,UAAU,CAAC,QAAQ,CAAC,EAAE,CAAC;QAC7B,MAAM,IAAI,KAAK,CAAC,mBAAmB,QAAQ,EAAE,CAAC,CAAC;IACjD,CAAC;IAED,MAAM,YAAY,GAAG,IAAI,CAAC,KAAK,CAAC,EAAE,CAAC,YAAY,CAAC,QAAQ,EAAE,OAAO,CAAC,CAAC,CAAC;IACpE,MAAM,MAAM,GAAG,OAAO,CAAC,YAAY,EAAE,EAAE,UAAU,EAAE,IAAI,EAAE,CAAC,CAAC;IAC3D,MAAM,eAAe,GAAG,UAAU,IAAI,QAAQ,CAAC;IAE/C,EAAE,CAAC,aAAa,CACd,eAAe,EACf,IAAI,CAAC,SAAS,CAAC,YAAY,EAAE,IAAI,EAAE,CAAC,CAAC,EACrC,OAAO,CACR,CAAC;IAEF,OAAO;QACL,GAAG,MAAM;QACT,eAAe,EAAE,eAAe;KACjC,CAAC;AACJ,CAAC;AAED;;GAEG;AACH,MAAM,UAAU,YAAY,CAAC,MAAwB;IACnD,MAAM,KAAK,GAAG;QACZ,GAAG,CAAC,MAAM,CAAC,EAAE,CAAC;QACd,oBAAoB;QACpB,GAAG,CAAC,MAAM,CAAC,EAAE,CAAC;QACd,oBAAoB,MAAM,CAAC,UAAU,EAAE;QACvC,oBAAoB,MAAM,CAAC,UAAU,EAAE;QACvC,oBAAoB,MAAM,CAAC,OAAO,EAAE;QACpC,oBAAoB,MAAM,CAAC,iBAAiB,CAAC,OAAO,CAAC,CAAC,CAAC,aAAa;KACrE,CAAC;IAEF,IAAI,MAAM,CAAC,cAAc,EAAE,CAAC;QAC1B,KAAK,CAAC,IAAI,CAAC,oBAAoB,MAAM,CAAC,cAAc,EAAE,CAAC,CAAC;IAC1D,CAAC;IAED,KAAK,CAAC,IAAI,CAAC,GAAG,CAAC,MAAM,CAAC,EAAE,CAAC,CAAC,CAAC;IAC3B,OAAO,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;AAC1B,CAAC;AAED,mBAAmB;AACnB,SAAS,eAAe,CAAC,GAAW;IAClC,IAAI,CAAC,EAAE,CAAC,UAAU,CAAC,GAAG,CAAC,EAAE,CAAC;QACxB,EAAE,CAAC,SAAS,CAAC,GAAG,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,CAAC;IACzC,CAAC;AACH,CAAC;AAED,SAAS,kBAAkB,CAAC,SAAiB,EAAE,QAAiB;IAC9D,MAAM,SAAS,GAAG,IAAI,IAAI,EAAE,CAAC,WAAW,EAAE,CAAC,OAAO,CAAC,OAAO,EAAE,GAAG,CAAC,CAAC;IACjE,MAAM,IAAI,GAAG,QAAQ,IAAI,MAAM,CAAC;IAChC,MAAM,aAAa,GAAG,GAAG,IAAI,IAAI,SAAS,OAAO,CAAC;IAClD,OAAO,IAAI,CAAC,IAAI,CAAC,SAAS,EAAE,aAAa,CAAC,CAAC;AAC7C,CAAC;AAED,OAAO,EAAE,MAAM,IAAI,MAAM,EAAE,MAAM,mBAAmB,CAAC"}
|
package/dist/types.d.ts
ADDED
|
@@ -0,0 +1,15 @@
|
|
|
1
|
+
export interface ComparisonResult {
|
|
2
|
+
jsonTokens: number;
|
|
3
|
+
toonTokens: number;
|
|
4
|
+
savings: number;
|
|
5
|
+
savingsPercentage: number;
|
|
6
|
+
toonOutput: string;
|
|
7
|
+
toonOutputPath?: string;
|
|
8
|
+
}
|
|
9
|
+
export interface CompareOptions {
|
|
10
|
+
outputDir?: string;
|
|
11
|
+
saveToFile?: boolean;
|
|
12
|
+
prettyJson?: boolean;
|
|
13
|
+
fileName?: string;
|
|
14
|
+
}
|
|
15
|
+
//# sourceMappingURL=types.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"types.d.ts","sourceRoot":"","sources":["../src/types.ts"],"names":[],"mappings":"AAAA,MAAM,WAAW,gBAAgB;IAC/B,UAAU,EAAE,MAAM,CAAC;IACnB,UAAU,EAAE,MAAM,CAAC;IACnB,OAAO,EAAE,MAAM,CAAC;IAChB,iBAAiB,EAAE,MAAM,CAAC;IAC1B,UAAU,EAAE,MAAM,CAAC;IACnB,cAAc,CAAC,EAAE,MAAM,CAAC;CACzB;AAED,MAAM,WAAW,cAAc;IAC7B,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,UAAU,CAAC,EAAE,OAAO,CAAC;IACrB,UAAU,CAAC,EAAE,OAAO,CAAC;IACrB,QAAQ,CAAC,EAAE,MAAM,CAAC;CACnB"}
|
package/dist/types.js
ADDED
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"types.js","sourceRoot":"","sources":["../src/types.ts"],"names":[],"mappings":""}
|
package/package.json
ADDED
|
@@ -0,0 +1,56 @@
|
|
|
1
|
+
{
|
|
2
|
+
"name": "json-toon-parser",
|
|
3
|
+
"version": "1.0.0",
|
|
4
|
+
"description": "Compare and convert JSON to TOON format with token efficiency analysis",
|
|
5
|
+
"main": "./dist/index.js",
|
|
6
|
+
"types": "./dist/index.d.ts",
|
|
7
|
+
"bin": {
|
|
8
|
+
"json-toon": "./dist/cli.js"
|
|
9
|
+
},
|
|
10
|
+
"type": "module",
|
|
11
|
+
"scripts": {
|
|
12
|
+
"build": "tsc",
|
|
13
|
+
"start": "npm run build && node dist/cli.js update data/data.json",
|
|
14
|
+
"prepublishOnly": "npm run build",
|
|
15
|
+
"test": "echo \"No tests specified\" && exit 0"
|
|
16
|
+
},
|
|
17
|
+
"files": [
|
|
18
|
+
"dist",
|
|
19
|
+
"README.md",
|
|
20
|
+
"LICENSE"
|
|
21
|
+
],
|
|
22
|
+
"keywords": [
|
|
23
|
+
"json",
|
|
24
|
+
"toon",
|
|
25
|
+
"parser",
|
|
26
|
+
"converter",
|
|
27
|
+
"tokenization",
|
|
28
|
+
"gpt",
|
|
29
|
+
"compression",
|
|
30
|
+
"format",
|
|
31
|
+
"efficiency",
|
|
32
|
+
"token-optimization"
|
|
33
|
+
],
|
|
34
|
+
"author": "Bhushan Chaudhari <bhushanschaudhari4@gmail.com>",
|
|
35
|
+
"license": "MIT",
|
|
36
|
+
"repository": {
|
|
37
|
+
"type": "git",
|
|
38
|
+
"url": "https://github.com/bhushan44/json-toon-parser.git"
|
|
39
|
+
},
|
|
40
|
+
"bugs": {
|
|
41
|
+
"url": "https://github.com/bhushan44/json-toon-parser/issues"
|
|
42
|
+
},
|
|
43
|
+
"homepage": "https://github.com/bhushan44/json-toon-parser#readme",
|
|
44
|
+
"dependencies": {
|
|
45
|
+
"@toon-format/toon": "^1.0.0",
|
|
46
|
+
"gpt-3-encoder": "^1.1.4",
|
|
47
|
+
"commander": "^12.0.0"
|
|
48
|
+
},
|
|
49
|
+
"devDependencies": {
|
|
50
|
+
"@types/node": "^20.0.0",
|
|
51
|
+
"typescript": "^5.0.0"
|
|
52
|
+
},
|
|
53
|
+
"engines": {
|
|
54
|
+
"node": ">=18.0.0"
|
|
55
|
+
}
|
|
56
|
+
}
|