@sablier/devkit 1.7.0 → 1.8.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/just/csv.just +92 -66
- package/just/tsv.just +2 -2
- package/package.json +1 -1
package/just/csv.just
CHANGED
|
@@ -5,72 +5,98 @@ import "./settings.just"
|
|
|
5
5
|
# ---------------------------------------------------------------------------- #
|
|
6
6
|
|
|
7
7
|
# Check CSV/TSV files using qsv: https://github.com/dathere/qsv
|
|
8
|
-
[group("checks"), script("
|
|
9
|
-
_csv-check globs="data/*/*.csv" schema="" ignore="*.csv.invalid|*.csv.valid|*validation-errors.csv"
|
|
10
|
-
|
|
11
|
-
|
|
12
|
-
|
|
13
|
-
|
|
14
|
-
|
|
15
|
-
|
|
16
|
-
|
|
17
|
-
|
|
18
|
-
|
|
19
|
-
|
|
20
|
-
|
|
21
|
-
|
|
22
|
-
|
|
23
|
-
|
|
24
|
-
|
|
25
|
-
|
|
26
|
-
|
|
27
|
-
|
|
28
|
-
|
|
29
|
-
|
|
30
|
-
|
|
31
|
-
|
|
32
|
-
|
|
33
|
-
|
|
34
|
-
|
|
35
|
-
|
|
36
|
-
|
|
37
|
-
|
|
38
|
-
|
|
8
|
+
[group("checks"), script("python3")]
|
|
9
|
+
_csv-check globs="data/*/*.csv" ext="CSV" schema="" ignore="*.csv.invalid|*.csv.valid|*validation-errors.csv":
|
|
10
|
+
import fnmatch
|
|
11
|
+
import glob as globmod
|
|
12
|
+
import subprocess
|
|
13
|
+
import sys
|
|
14
|
+
|
|
15
|
+
# Check qsv is available
|
|
16
|
+
if subprocess.run(["which", "qsv"], capture_output=True).returncode != 0:
|
|
17
|
+
print("✗ qsv CLI not found")
|
|
18
|
+
print("Install it: https://github.com/dathere/qsv")
|
|
19
|
+
sys.exit(1)
|
|
20
|
+
|
|
21
|
+
ext = "{{ ext }}" or "CSV"
|
|
22
|
+
ignore_patterns = "{{ ignore }}".split("|") if "{{ ignore }}" else []
|
|
23
|
+
schema = "{{ schema }}" or None
|
|
24
|
+
globs = "{{ globs }}"
|
|
25
|
+
|
|
26
|
+
print(f"Validating {ext} files...")
|
|
27
|
+
files = globmod.glob(globs)
|
|
28
|
+
|
|
29
|
+
# Filter ignored files
|
|
30
|
+
files = [f for f in files if not any(fnmatch.fnmatch(f, p) for p in ignore_patterns)]
|
|
31
|
+
|
|
32
|
+
if not files:
|
|
33
|
+
print(f"ℹ️ No {ext} files found to validate")
|
|
34
|
+
sys.exit(0)
|
|
35
|
+
|
|
36
|
+
for file in files:
|
|
37
|
+
cmd = ["qsv", "validate", file]
|
|
38
|
+
if schema:
|
|
39
|
+
cmd.append(schema)
|
|
40
|
+
result = subprocess.run(cmd, capture_output=True)
|
|
41
|
+
if result.returncode != 0:
|
|
42
|
+
print(f"❌ Validation failed for: {file}")
|
|
43
|
+
subprocess.run(["just", "_csv-show-errors", file, ext])
|
|
44
|
+
sys.exit(1)
|
|
45
|
+
|
|
46
|
+
print(f"✅ All {ext} files are valid")
|
|
39
47
|
|
|
40
48
|
# Show validation errors for a CSV/TSV file
|
|
41
|
-
[group("checks"), script("
|
|
49
|
+
[group("checks"), script("python3")]
|
|
42
50
|
_csv-show-errors file ext="CSV":
|
|
43
|
-
|
|
44
|
-
|
|
45
|
-
|
|
46
|
-
|
|
47
|
-
|
|
48
|
-
|
|
49
|
-
|
|
50
|
-
|
|
51
|
-
|
|
52
|
-
|
|
53
|
-
|
|
54
|
-
|
|
55
|
-
|
|
56
|
-
|
|
57
|
-
|
|
58
|
-
|
|
59
|
-
|
|
60
|
-
|
|
61
|
-
|
|
62
|
-
|
|
63
|
-
|
|
64
|
-
|
|
65
|
-
|
|
66
|
-
|
|
67
|
-
|
|
68
|
-
|
|
69
|
-
|
|
70
|
-
|
|
71
|
-
|
|
72
|
-
|
|
73
|
-
|
|
74
|
-
|
|
75
|
-
|
|
76
|
-
|
|
51
|
+
import os
|
|
52
|
+
import subprocess
|
|
53
|
+
import sys
|
|
54
|
+
|
|
55
|
+
file = "{{ file }}"
|
|
56
|
+
error_file = f"{file}.validation-errors.tsv"
|
|
57
|
+
|
|
58
|
+
if not os.path.exists(error_file):
|
|
59
|
+
print(f"Error file not found: {error_file}")
|
|
60
|
+
sys.exit(0)
|
|
61
|
+
|
|
62
|
+
# Count errors (qsv auto-detects .tsv delimiter)
|
|
63
|
+
result = subprocess.run(["qsv", "count", error_file], capture_output=True, text=True)
|
|
64
|
+
try:
|
|
65
|
+
total = int(result.stdout.strip()) if result.returncode == 0 and result.stdout.strip() else 0
|
|
66
|
+
except ValueError:
|
|
67
|
+
total = 0
|
|
68
|
+
|
|
69
|
+
print()
|
|
70
|
+
if total > 0:
|
|
71
|
+
print(f"First 20 validation errors ({total} total):")
|
|
72
|
+
else:
|
|
73
|
+
print("Validation errors:")
|
|
74
|
+
print()
|
|
75
|
+
|
|
76
|
+
# Try qsv table for nice formatting, fall back to reading file directly
|
|
77
|
+
slice_result = subprocess.run(
|
|
78
|
+
["qsv", "slice", "--start", "0", "--len", "21", error_file],
|
|
79
|
+
capture_output=True, text=True
|
|
80
|
+
)
|
|
81
|
+
if slice_result.returncode == 0:
|
|
82
|
+
table_result = subprocess.run(
|
|
83
|
+
["qsv", "table"],
|
|
84
|
+
input=slice_result.stdout,
|
|
85
|
+
capture_output=True, text=True
|
|
86
|
+
)
|
|
87
|
+
if table_result.returncode == 0:
|
|
88
|
+
print(table_result.stdout)
|
|
89
|
+
else:
|
|
90
|
+
print(slice_result.stdout)
|
|
91
|
+
else:
|
|
92
|
+
with open(error_file) as f:
|
|
93
|
+
for i, line in enumerate(f):
|
|
94
|
+
if i >= 21:
|
|
95
|
+
break
|
|
96
|
+
print(line, end="")
|
|
97
|
+
|
|
98
|
+
if total > 20:
|
|
99
|
+
print()
|
|
100
|
+
print(f"... and {total - 20} more errors")
|
|
101
|
+
print()
|
|
102
|
+
print(f"Full details: {error_file}")
|
package/just/tsv.just
CHANGED
|
@@ -6,8 +6,8 @@ import "./csv.just"
|
|
|
6
6
|
|
|
7
7
|
# Check TSV files using qsv: https://github.com/dathere/qsv
|
|
8
8
|
[group("checks")]
|
|
9
|
-
_tsv-check globs="data/*/*.tsv" schema="" ignore="*.tsv.invalid|*.tsv.valid|*validation-errors.tsv":
|
|
10
|
-
@just _csv-check "{{ globs }}" "{{
|
|
9
|
+
_tsv-check globs="data/*/*.tsv" ext="TSV" schema="" ignore="*.tsv.invalid|*.tsv.valid|*validation-errors.tsv":
|
|
10
|
+
@just _csv-check "{{ globs }}" "{{ ext }}" "{{ schema }}" "{{ ignore }}"
|
|
11
11
|
|
|
12
12
|
# Show validation errors for a TSV file
|
|
13
13
|
[group("checks")]
|
package/package.json
CHANGED