champollion 0.3.3
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +133 -0
- package/README.md +387 -0
- package/bin/cli.js +278 -0
- package/index.js +135 -0
- package/lib/api-key.js +127 -0
- package/lib/autofix.js +432 -0
- package/lib/bridge/method_bridge.py +430 -0
- package/lib/card-source-resolution.mjs +284 -0
- package/lib/cards/cache.js +169 -0
- package/lib/cards/env.js +82 -0
- package/lib/cards/fetch-card-child.js +38 -0
- package/lib/cards/reader.js +435 -0
- package/lib/cards/refresh.js +111 -0
- package/lib/cards/remote.js +387 -0
- package/lib/cldf-export.mjs +540 -0
- package/lib/cldf-terms.mjs +62 -0
- package/lib/command-help.js +790 -0
- package/lib/commands/audit.js +49 -0
- package/lib/commands/card.js +454 -0
- package/lib/commands/doctor.js +559 -0
- package/lib/commands/fonts.js +489 -0
- package/lib/commands/help.js +91 -0
- package/lib/commands/init.js +1259 -0
- package/lib/commands/integrity.js +148 -0
- package/lib/commands/leaderboard.js +478 -0
- package/lib/commands/lint.js +30 -0
- package/lib/commands/models.js +177 -0
- package/lib/commands/plugin.js +103 -0
- package/lib/commands/provenance.js +45 -0
- package/lib/commands/recommend.js +75 -0
- package/lib/commands/register-corpus.js +678 -0
- package/lib/commands/repair-script.js +42 -0
- package/lib/commands/seal-corpus.js +355 -0
- package/lib/commands/seo.js +72 -0
- package/lib/commands/serve.js +147 -0
- package/lib/commands/status.js +265 -0
- package/lib/commands/submit.js +332 -0
- package/lib/commands/sync.js +89 -0
- package/lib/commands/tm.js +573 -0
- package/lib/commands/verify.js +39 -0
- package/lib/commands/watch.js +20 -0
- package/lib/commands/wrap.js +138 -0
- package/lib/commands/xliff.js +327 -0
- package/lib/commercial-eligibility.js +235 -0
- package/lib/concurrent.js +87 -0
- package/lib/config.js +523 -0
- package/lib/contamination-lane.js +76 -0
- package/lib/content-sync.js +731 -0
- package/lib/content.js +733 -0
- package/lib/corpus-registration.mjs +608 -0
- package/lib/cost-report.js +346 -0
- package/lib/diff.js +155 -0
- package/lib/docusaurus-sync.js +1256 -0
- package/lib/flatten.js +55 -0
- package/lib/format.js +954 -0
- package/lib/hash.js +159 -0
- package/lib/icu.js +473 -0
- package/lib/integrity.js +689 -0
- package/lib/license-gate.mjs +478 -0
- package/lib/license-identify.mjs +229 -0
- package/lib/lint.js +629 -0
- package/lib/method-manifest.js +60 -0
- package/lib/methods/anthropic.js +140 -0
- package/lib/methods/apertium.js +163 -0
- package/lib/methods/api.js +316 -0
- package/lib/methods/base.js +184 -0
- package/lib/methods/content-separator.js +45 -0
- package/lib/methods/deepl.js +426 -0
- package/lib/methods/direct-llm.js +586 -0
- package/lib/methods/external.js +332 -0
- package/lib/methods/fetch-with-retry.js +124 -0
- package/lib/methods/gemini.js +147 -0
- package/lib/methods/google-translate.js +402 -0
- package/lib/methods/http-utils.js +122 -0
- package/lib/methods/libretranslate.js +314 -0
- package/lib/methods/llm-coached.js +670 -0
- package/lib/methods/llm.js +592 -0
- package/lib/methods/local.js +76 -0
- package/lib/methods/microsoft-translator.js +331 -0
- package/lib/methods/openai.js +131 -0
- package/lib/methods/openrouter-client.js +327 -0
- package/lib/methods/openrouter-pricing.js +156 -0
- package/lib/methods/provider-env.js +115 -0
- package/lib/methods/provider-pricing.js +310 -0
- package/lib/methods/tilde.js +150 -0
- package/lib/methods/translated.js +229 -0
- package/lib/methods/translation-error.js +80 -0
- package/lib/models.js +258 -0
- package/lib/no-translate.js +233 -0
- package/lib/output.js +238 -0
- package/lib/pairs.js +547 -0
- package/lib/plugins.js +447 -0
- package/lib/provenance.js +323 -0
- package/lib/recommend.js +648 -0
- package/lib/registers.js +1185 -0
- package/lib/repair-script.js +266 -0
- package/lib/scripts.js +994 -0
- package/lib/seal.mjs +464 -0
- package/lib/sealed-qualifier.mjs +211 -0
- package/lib/security.js +59 -0
- package/lib/segment.js +369 -0
- package/lib/seo.js +275 -0
- package/lib/serve.js +854 -0
- package/lib/string-classify.js +85 -0
- package/lib/submit.mjs +344 -0
- package/lib/sync.js +969 -0
- package/lib/tags/bcp47.js +202 -0
- package/lib/tags/resolve.js +314 -0
- package/lib/terminology.js +111 -0
- package/lib/tm-seed.js +294 -0
- package/lib/tm.js +515 -0
- package/lib/translate-pair.js +197 -0
- package/lib/translate.js +203 -0
- package/lib/types.js +230 -0
- package/lib/validate.js +510 -0
- package/lib/verify.js +451 -0
- package/lib/watch.js +145 -0
- package/lib/xliff.js +184 -0
- package/package.json +93 -0
- package/shared/ATTRIBUTION.md +145 -0
- package/shared/CORPORA-CARDS.md +288 -0
- package/shared/DATA-SOVEREIGNTY.md +500 -0
- package/shared/LANGUAGE-CARD-FIELDS.md +532 -0
- package/shared/card-lint-baseline.json +3189 -0
- package/shared/cards-fallback.json +1 -0
- package/shared/catalogue/card-config.json +6091 -0
- package/shared/catalogue/external-results.json +3888 -0
- package/shared/catalogue/gender-guidance.json +1038 -0
- package/shared/catalogue/method-coverage.json +1751 -0
- package/shared/catalogue/metric-coverage.json +170 -0
- package/shared/catalogue/metric-reliability.json +1 -0
- package/shared/catalogue/register-presets.json +3180 -0
- package/shared/catalogue/vitality-scales.json +55 -0
- package/shared/cldr-index.json +1115 -0
- package/shared/code-bridge.json +253 -0
- package/shared/corpora-cards-v1-reference.md +281 -0
- package/shared/curated-dictionary-flags.json +35 -0
- package/shared/curated-endonyms.json +35 -0
- package/shared/curated-fsts.json +51 -0
- package/shared/curated-orthography-conventions.json +26 -0
- package/shared/curated-sil-resources.json +374 -0
- package/shared/curated-tools.json +41 -0
- package/shared/docent/corpus.json +11333 -0
- package/shared/docent/faq.en.json +564 -0
- package/shared/docent/register-blocks.json +60 -0
- package/shared/docent/system-prompt.md +144 -0
- package/shared/domain-taxonomy.json +35 -0
- package/shared/explainers/glossary.json +2975 -0
- package/shared/explainers/tc-features.json +20112 -0
- package/shared/explainers/term-watchlist.json +147 -0
- package/shared/human-services.json +59 -0
- package/shared/license-corrections.json +261 -0
- package/shared/license-evidence.json +13452 -0
- package/shared/licenses.json +6781 -0
- package/shared/method-registry.json +236 -0
- package/shared/metric-registry.json +620 -0
- package/shared/model-aliases.json +7 -0
- package/shared/schemas/champollion-plugin.schema.json +206 -0
- package/shared/schemas/corpora-card.schema.json +957 -0
- package/shared/schemas/domain-taxonomy.schema.json +64 -0
- package/shared/schemas/external-results.schema.json +314 -0
- package/shared/schemas/human-services.schema.json +90 -0
- package/shared/schemas/language-card.schema.json +1308 -0
- package/shared/schemas/licenses.schema.json +155 -0
- package/shared/schemas/method-card.schema.json +412 -0
- package/shared/schemas/method-registry.schema.json +85 -0
- package/shared/schemas/metric-registry.schema.json +96 -0
- package/shared/schemas/metric-reliability.schema.json +178 -0
- package/shared/schemas/model-aliases.schema.json +27 -0
- package/shared/schemas/source-snapshot.schema.json +96 -0
|
@@ -0,0 +1,148 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Command: integrity
|
|
3
|
+
*
|
|
4
|
+
* Audits locale files for format, encoding, and placeholder consistency.
|
|
5
|
+
* Catches: mismatched {placeholders}, encoding corruption, untranslated
|
|
6
|
+
* copies, and orphan keys not in the source file.
|
|
7
|
+
*
|
|
8
|
+
* Returns exit code 1 if issues found (unless --warn-only is set).
|
|
9
|
+
*/
|
|
10
|
+
|
|
11
|
+
import fs from 'node:fs';
|
|
12
|
+
import path from 'node:path';
|
|
13
|
+
import { resolveConfig } from '../config.js';
|
|
14
|
+
import { auditLocalePair, formatIntegrityReport } from '../integrity.js';
|
|
15
|
+
import { compileNoTranslate } from '../no-translate.js';
|
|
16
|
+
import { resolvePairs } from '../pairs.js';
|
|
17
|
+
import { loadTM, lookupTM, tmMethodKey } from '../tm.js';
|
|
18
|
+
import { getLanguageCard } from '../registers.js';
|
|
19
|
+
import { SCRIPT_CONVERTERS, converterKeyForLocale } from '../scripts.js';
|
|
20
|
+
import { flattenKeys } from '../flatten.js';
|
|
21
|
+
import { readLocaleFile, detectFormatFromDir, getExtension } from '../format.js';
|
|
22
|
+
import { output } from '../output.js';
|
|
23
|
+
|
|
24
|
+
/**
|
|
25
|
+
* @param {import('../types.js').CLIArgs} args - Parsed CLI arguments
|
|
26
|
+
* @param {string} cwd - Working directory
|
|
27
|
+
* @returns {Promise<number>} Exit code (0 = success, 1 = error)
|
|
28
|
+
*/
|
|
29
|
+
async function run(args, cwd) {
|
|
30
|
+
// --json: stdout carries exactly one JSON document. Quiet mode keeps the
|
|
31
|
+
// human report lines off stdout; errors still reach stderr.
|
|
32
|
+
const json = !!args.json;
|
|
33
|
+
if (json) output.setMode('quiet');
|
|
34
|
+
|
|
35
|
+
const config = resolveConfig(args, cwd);
|
|
36
|
+
const noTranslate = compileNoTranslate(config);
|
|
37
|
+
|
|
38
|
+
// Per-locale script expectations for the unexpected-PUA check. Only locales
|
|
39
|
+
// whose converter emits PUA matter here; the resolution says whether that
|
|
40
|
+
// PUA is the configured output (converts) or damage. resolvePairs does not
|
|
41
|
+
// throw on the choice-required state, so auditing an unconfigured crk/sr
|
|
42
|
+
// project still works.
|
|
43
|
+
const scriptExpectations = new Map();
|
|
44
|
+
// TM method key per locale — echoes the TM confirms as pipeline-produced
|
|
45
|
+
// are settled, not issues. This is the SAME suppression the sync diff
|
|
46
|
+
// applies; without it, integrity reported thousands of "untranslated
|
|
47
|
+
// copies" on a project sync calls fully synced. Read-only TM load.
|
|
48
|
+
const tm = loadTM(cwd);
|
|
49
|
+
const tmKeys = new Map();
|
|
50
|
+
for (const [, pc] of resolvePairs(config)) {
|
|
51
|
+
tmKeys.set(pc.target, tmMethodKey(pc));
|
|
52
|
+
const key = converterKeyForLocale(pc.target, getLanguageCard(pc.target));
|
|
53
|
+
if (!key || !SCRIPT_CONVERTERS[key].puaRange) continue;
|
|
54
|
+
scriptExpectations.set(pc.target, {
|
|
55
|
+
locale: pc.target,
|
|
56
|
+
converts: !!pc.scriptResolution?.converterKey,
|
|
57
|
+
});
|
|
58
|
+
}
|
|
59
|
+
|
|
60
|
+
const format = config.format !== 'auto'
|
|
61
|
+
? config.format
|
|
62
|
+
: detectFormatFromDir(config.localesDir);
|
|
63
|
+
const ext = getExtension(format);
|
|
64
|
+
const sourcePath = path.join(config.localesDir, `${config.inputLocale}${ext}`);
|
|
65
|
+
|
|
66
|
+
if (!fs.existsSync(sourcePath)) {
|
|
67
|
+
if (json) {
|
|
68
|
+
console.log(JSON.stringify({
|
|
69
|
+
command: 'integrity',
|
|
70
|
+
error: `Source locale file not found: ${sourcePath}`,
|
|
71
|
+
}, null, 2));
|
|
72
|
+
return 1;
|
|
73
|
+
}
|
|
74
|
+
output.error(`Source locale file not found: ${sourcePath}`);
|
|
75
|
+
return 1;
|
|
76
|
+
}
|
|
77
|
+
|
|
78
|
+
const sourceRaw = readLocaleFile(sourcePath, format);
|
|
79
|
+
const sourceFlat = format === 'json' ? flattenKeys(sourceRaw) : sourceRaw;
|
|
80
|
+
|
|
81
|
+
// Detect target locales from directory listing
|
|
82
|
+
const files = fs.readdirSync(config.localesDir);
|
|
83
|
+
const targetLocales = files
|
|
84
|
+
.filter(f => f.endsWith(ext) && !f.startsWith(config.inputLocale))
|
|
85
|
+
.map(f => f.replace(ext, ''));
|
|
86
|
+
|
|
87
|
+
output.raw('\n champollion integrity — Locale File Audit\n');
|
|
88
|
+
output.raw(` Source: ${config.inputLocale} (${Object.keys(sourceFlat).length} keys)`);
|
|
89
|
+
output.raw(` Targets: ${targetLocales.join(', ')}\n`);
|
|
90
|
+
|
|
91
|
+
let totalIssues = 0;
|
|
92
|
+
const localeReports = [];
|
|
93
|
+
|
|
94
|
+
for (const locale of targetLocales) {
|
|
95
|
+
const targetPath = path.join(config.localesDir, `${locale}${ext}`);
|
|
96
|
+
const targetRaw = readLocaleFile(targetPath, format);
|
|
97
|
+
const targetFlat = format === 'json' ? flattenKeys(targetRaw) : targetRaw;
|
|
98
|
+
|
|
99
|
+
const tmKey = tmKeys.get(locale) || null;
|
|
100
|
+
const audit = auditLocalePair(sourceFlat, targetFlat, locale, {
|
|
101
|
+
noTranslate,
|
|
102
|
+
scriptExpectation: scriptExpectations.get(locale) || null,
|
|
103
|
+
isConfirmedEcho: tmKey
|
|
104
|
+
? (key, sourceValue) => lookupTM(tm, sourceValue, locale, tmKey) === sourceValue
|
|
105
|
+
: null,
|
|
106
|
+
});
|
|
107
|
+
const report = formatIntegrityReport(locale, audit);
|
|
108
|
+
output.raw(report);
|
|
109
|
+
|
|
110
|
+
// Only these five categories drive the exit code (pluralIssues and
|
|
111
|
+
// bomFiles are reported but advisory) — issueCount matches that.
|
|
112
|
+
//
|
|
113
|
+
// noTranslateDrift is an error, not advisory: a declared-verbatim key
|
|
114
|
+
// that isn't verbatim is a broken value in a shipped locale file (the
|
|
115
|
+
// corrupted-URL class), and `champollion sync` repairs it deterministically.
|
|
116
|
+
// unexpectedPua likewise: PUA with conversion off renders blank, and
|
|
117
|
+
// `champollion repair-script` repairs it deterministically.
|
|
118
|
+
// hollowedValues likewise: old-pipeline damage that sync considers
|
|
119
|
+
// settled — only a forced re-translation fixes it, and only this audit
|
|
120
|
+
// surfaces it.
|
|
121
|
+
const issueCount = audit.placeholderIssues.length +
|
|
122
|
+
audit.encodingIssues.length +
|
|
123
|
+
audit.copies.length +
|
|
124
|
+
audit.orphans.length +
|
|
125
|
+
audit.noTranslateDrift.length +
|
|
126
|
+
audit.unexpectedPua.length +
|
|
127
|
+
audit.hollowedValues.length;
|
|
128
|
+
totalIssues += issueCount;
|
|
129
|
+
localeReports.push({ locale, issues: audit, issueCount });
|
|
130
|
+
}
|
|
131
|
+
|
|
132
|
+
output.raw(` Total issues: ${totalIssues}`);
|
|
133
|
+
|
|
134
|
+
const exitCode = (totalIssues > 0 && !args['warn-only']) ? 1 : 0;
|
|
135
|
+
if (json) {
|
|
136
|
+
console.log(JSON.stringify({
|
|
137
|
+
command: 'integrity',
|
|
138
|
+
source: config.inputLocale,
|
|
139
|
+
sourceKeys: Object.keys(sourceFlat).length,
|
|
140
|
+
locales: localeReports,
|
|
141
|
+
totalIssues,
|
|
142
|
+
warnOnly: !!args['warn-only'],
|
|
143
|
+
}, null, 2));
|
|
144
|
+
}
|
|
145
|
+
return exitCode;
|
|
146
|
+
}
|
|
147
|
+
|
|
148
|
+
export { run };
|
|
@@ -0,0 +1,478 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Command: leaderboard
|
|
3
|
+
*
|
|
4
|
+
* Fetches and displays MT evaluation leaderboard data from Supabase.
|
|
5
|
+
* Provides sorting, filtering, JSON export, and method installation.
|
|
6
|
+
*
|
|
7
|
+
* Usage:
|
|
8
|
+
* champollion leaderboard # Show all results
|
|
9
|
+
* champollion leaderboard --pair "en>crk" # Filter by language pair (quote the >)
|
|
10
|
+
* champollion leaderboard --sort composite # Sort by composite score
|
|
11
|
+
* champollion leaderboard --json # Machine-readable NDJSON
|
|
12
|
+
* champollion leaderboard --top 5 # Show top N results
|
|
13
|
+
* champollion leaderboard --install 1 # Install method config from rank 1
|
|
14
|
+
* champollion leaderboard --install 1 --apply # Install and wire to config
|
|
15
|
+
*/
|
|
16
|
+
|
|
17
|
+
import { output } from '../output.js';
|
|
18
|
+
import { getLanguageCard } from '../registers.js';
|
|
19
|
+
// Supabase public config — shared with the dynamic card loader
|
|
20
|
+
// (RLS restricts the anon role to read-only)
|
|
21
|
+
import { SUPABASE_URL, SUPABASE_ANON_KEY } from '../cards/env.js';
|
|
22
|
+
import {
|
|
23
|
+
LANE_ABSOLUTE, LANE_RELATIVE_ONLY, normalizeGrade, isRelativeOnly,
|
|
24
|
+
} from '../contamination-lane.js';
|
|
25
|
+
import fs from 'node:fs';
|
|
26
|
+
import path from 'node:path';
|
|
27
|
+
|
|
28
|
+
/**
|
|
29
|
+
* Sortable metric definitions.
|
|
30
|
+
* key: CLI flag value → column: Supabase column → label: Display header
|
|
31
|
+
*/
|
|
32
|
+
const SORT_KEYS = {
|
|
33
|
+
composite: { column: 'composite_score', label: 'Composite', desc: true },
|
|
34
|
+
chrf: { column: 'chrf_plus_plus', label: 'chrF++', desc: true },
|
|
35
|
+
exact: { column: 'exact_match_rate', label: 'Exact Match', desc: true },
|
|
36
|
+
fst: { column: 'fst_acceptance_rate', label: 'FST Accept', desc: true },
|
|
37
|
+
equivalent: { column: 'equivalent_match_rate', label: 'Equiv Match', desc: true },
|
|
38
|
+
semantic: { column: 'semantic_score', label: 'Semantic', desc: true },
|
|
39
|
+
cost: { column: 'total_cost_usd', label: 'Cost (USD)', desc: false },
|
|
40
|
+
date: { column: 'run_timestamp', label: 'Date', desc: true },
|
|
41
|
+
};
|
|
42
|
+
|
|
43
|
+
/**
|
|
44
|
+
* Quality tier → display label with color hint.
|
|
45
|
+
*/
|
|
46
|
+
const TIER_LABELS = {
|
|
47
|
+
baseline: '○ Baseline',
|
|
48
|
+
emerging: '◔ Emerging',
|
|
49
|
+
functional: '◑ Functional',
|
|
50
|
+
deployable: '◕ Deployable',
|
|
51
|
+
fluent: '● Fluent',
|
|
52
|
+
};
|
|
53
|
+
|
|
54
|
+
// ---------------------------------------------------------------------------
|
|
55
|
+
// Contamination lane — FAIL SAFE. The lane policy lives in
|
|
56
|
+
// lib/contamination-lane.js (the CLI-side mirror of the SSOT in
|
|
57
|
+
// arena/mt_eval_harness/contamination.py).
|
|
58
|
+
//
|
|
59
|
+
// The CLI reads each run's grade from run_card.contamination (publish.py stamps
|
|
60
|
+
// it; there is no top-level column) — it does not fetch the datasets table, so
|
|
61
|
+
// a run with no stamped grade fails safe to relative-only rather than silently
|
|
62
|
+
// ranking as absolute quality. This is intentionally conservative.
|
|
63
|
+
// ---------------------------------------------------------------------------
|
|
64
|
+
|
|
65
|
+
/** Row's normalized contamination grade from run_card.contamination, or null. */
|
|
66
|
+
function rowContamination(row) {
|
|
67
|
+
return normalizeGrade(row?.run_card?.contamination);
|
|
68
|
+
}
|
|
69
|
+
|
|
70
|
+
/** FAIL SAFE: true ⇒ relative-comparison-only (do not co-rank as absolute
|
|
71
|
+
* quality). Known HIGH/MEDIUM or unknown grade → true; only LOW → false. */
|
|
72
|
+
function rowIsRelativeOnly(row) {
|
|
73
|
+
return isRelativeOnly(rowContamination(row));
|
|
74
|
+
}
|
|
75
|
+
|
|
76
|
+
/**
|
|
77
|
+
* Format a language pair code into display name using language cards.
|
|
78
|
+
* "en>crk" → "English → Plains Cree"
|
|
79
|
+
*/
|
|
80
|
+
function formatPairDisplay(pair) {
|
|
81
|
+
if (!pair) return '?';
|
|
82
|
+
const sep = pair.includes('>') ? '>' : ' → ';
|
|
83
|
+
const [src, tgt] = pair.split(sep);
|
|
84
|
+
if (!tgt) return pair;
|
|
85
|
+
|
|
86
|
+
const srcCard = getLanguageCard(src.trim());
|
|
87
|
+
const tgtCard = getLanguageCard(tgt.trim());
|
|
88
|
+
const srcName = srcCard?.name || src.trim().toUpperCase();
|
|
89
|
+
const tgtName = tgtCard?.name || tgt.trim().toUpperCase();
|
|
90
|
+
return `${srcName} → ${tgtName}`;
|
|
91
|
+
}
|
|
92
|
+
|
|
93
|
+
/**
|
|
94
|
+
* Format a metric value for table display.
|
|
95
|
+
*/
|
|
96
|
+
function fmtMetric(value, decimals = 2) {
|
|
97
|
+
if (value == null) return '—';
|
|
98
|
+
return Number(value).toFixed(decimals);
|
|
99
|
+
}
|
|
100
|
+
|
|
101
|
+
/**
|
|
102
|
+
* Pad a string to a fixed width for table alignment.
|
|
103
|
+
*/
|
|
104
|
+
function pad(str, width, align = 'left') {
|
|
105
|
+
const s = String(str);
|
|
106
|
+
if (s.length >= width) return s.slice(0, width);
|
|
107
|
+
const padding = ' '.repeat(width - s.length);
|
|
108
|
+
return align === 'right' ? padding + s : s + padding;
|
|
109
|
+
}
|
|
110
|
+
|
|
111
|
+
/**
|
|
112
|
+
* Fetch leaderboard data from Supabase.
|
|
113
|
+
*/
|
|
114
|
+
async function fetchLeaderboard(sortKey, pair) {
|
|
115
|
+
const sort = SORT_KEYS[sortKey] || SORT_KEYS.composite;
|
|
116
|
+
const order = `${sort.column}.${sort.desc ? 'desc' : 'asc'}.nullslast`;
|
|
117
|
+
|
|
118
|
+
let url = `${SUPABASE_URL}/rest/v1/run_cards?select=*&order=${order}`;
|
|
119
|
+
if (pair) {
|
|
120
|
+
url += `&language_pair=eq.${pair}`;
|
|
121
|
+
}
|
|
122
|
+
|
|
123
|
+
const resp = await fetch(url, {
|
|
124
|
+
headers: {
|
|
125
|
+
apikey: SUPABASE_ANON_KEY,
|
|
126
|
+
Authorization: `Bearer ${SUPABASE_ANON_KEY}`,
|
|
127
|
+
},
|
|
128
|
+
});
|
|
129
|
+
|
|
130
|
+
if (!resp.ok) {
|
|
131
|
+
throw new Error(`Supabase returned HTTP ${resp.status}: ${resp.statusText}`);
|
|
132
|
+
}
|
|
133
|
+
|
|
134
|
+
return resp.json();
|
|
135
|
+
}
|
|
136
|
+
|
|
137
|
+
/**
|
|
138
|
+
* @param {import('../types.js').CLIArgs} args - Parsed CLI arguments
|
|
139
|
+
* @param {string} cwd - Working directory
|
|
140
|
+
* @returns {Promise<number>} Exit code (0 = success, 1 = error)
|
|
141
|
+
*/
|
|
142
|
+
async function run(args, cwd) {
|
|
143
|
+
const sortKey = args.sort || 'composite';
|
|
144
|
+
const pair = args.pair || null;
|
|
145
|
+
const topN = args.top ? parseInt(args.top, 10) : null;
|
|
146
|
+
const jsonMode = args.json || false;
|
|
147
|
+
|
|
148
|
+
// Resolve --install into an integer rank. Because --install is a string flag,
|
|
149
|
+
// a bare `--install` (no rank) parses as boolean `true`, and parseInt(true)
|
|
150
|
+
// is NaN — which previously slipped past the range check in _handleInstall
|
|
151
|
+
// and crashed on `rows[NaN - 1].run_card`. Validate here: --install requires
|
|
152
|
+
// a positive-integer rank.
|
|
153
|
+
let installRank = null;
|
|
154
|
+
if (args.install != null && args.install !== false) {
|
|
155
|
+
installRank = parseInt(args.install, 10);
|
|
156
|
+
if (!Number.isInteger(installRank) || installRank < 1) {
|
|
157
|
+
output.error('--install requires a positive integer rank, e.g. `champollion leaderboard --install 1`.');
|
|
158
|
+
return 1;
|
|
159
|
+
}
|
|
160
|
+
}
|
|
161
|
+
|
|
162
|
+
// Validate sort key
|
|
163
|
+
if (!SORT_KEYS[sortKey]) {
|
|
164
|
+
output.error(`Unknown sort key: "${sortKey}"`);
|
|
165
|
+
output.error(`Valid keys: ${Object.keys(SORT_KEYS).join(', ')}`);
|
|
166
|
+
return 1;
|
|
167
|
+
}
|
|
168
|
+
|
|
169
|
+
try {
|
|
170
|
+
const rows = await fetchLeaderboard(sortKey, pair);
|
|
171
|
+
|
|
172
|
+
if (rows.length === 0) {
|
|
173
|
+
if (jsonMode) {
|
|
174
|
+
// Empty JSON array for machine consumers
|
|
175
|
+
console.log('[]');
|
|
176
|
+
} else {
|
|
177
|
+
output.raw('\n No leaderboard entries found.');
|
|
178
|
+
if (pair) output.raw(` (filtered by pair: ${pair})`);
|
|
179
|
+
output.raw(' Submit results with `mt-eval publish`.\n');
|
|
180
|
+
}
|
|
181
|
+
return 0;
|
|
182
|
+
}
|
|
183
|
+
|
|
184
|
+
// ---- --install mode: extract method config from a ranked entry ----
|
|
185
|
+
if (installRank != null) {
|
|
186
|
+
return _handleInstall(rows, installRank, sortKey, cwd, args);
|
|
187
|
+
}
|
|
188
|
+
|
|
189
|
+
// Apply --top limit
|
|
190
|
+
const display = topN ? rows.slice(0, topN) : rows;
|
|
191
|
+
|
|
192
|
+
// Lane warning (FAIL SAFE) — goes to stderr so it never pollutes --json
|
|
193
|
+
// stdout. A relative-comparison-only corpus (HIGH/MEDIUM contamination,
|
|
194
|
+
// FLORES, or unknown grade) is NOT absolute quality; flag it loudly when it
|
|
195
|
+
// surfaces at the top so the rank isn't misread as "best quality".
|
|
196
|
+
const relRows = display.filter(rowIsRelativeOnly);
|
|
197
|
+
if (relRows.length > 0) {
|
|
198
|
+
if (display.length > 0 && rowIsRelativeOnly(display[0])) {
|
|
199
|
+
output.warn(
|
|
200
|
+
`Rank #1 is a relative-comparison-only corpus `
|
|
201
|
+
+ `(${rowContamination(display[0]) || 'grade unknown'}) — its score is valid only `
|
|
202
|
+
+ `against other methods on that corpus, NOT as absolute quality.`,
|
|
203
|
+
);
|
|
204
|
+
}
|
|
205
|
+
output.warn(
|
|
206
|
+
`${relRows.length} of ${display.length} shown row(s) are relative-comparison-only `
|
|
207
|
+
+ `and must not be ranked against absolute-quality corpora.`,
|
|
208
|
+
);
|
|
209
|
+
}
|
|
210
|
+
|
|
211
|
+
if (jsonMode) {
|
|
212
|
+
// NDJSON output for CI/CD piping
|
|
213
|
+
for (const row of display) {
|
|
214
|
+
console.log(JSON.stringify({
|
|
215
|
+
rank: display.indexOf(row) + 1,
|
|
216
|
+
model: row.model_slug,
|
|
217
|
+
condition: row.condition,
|
|
218
|
+
pair: row.language_pair,
|
|
219
|
+
composite: row.composite_score,
|
|
220
|
+
chrF: row.chrf_plus_plus,
|
|
221
|
+
exactMatch: row.exact_match_rate,
|
|
222
|
+
fstAcceptance: row.fst_acceptance_rate,
|
|
223
|
+
equivalentMatch: row.equivalent_match_rate,
|
|
224
|
+
semanticScore: row.semantic_score,
|
|
225
|
+
tier: row.quality_tier,
|
|
226
|
+
cost_usd: row.total_cost_usd,
|
|
227
|
+
submitter: row.submitter,
|
|
228
|
+
date: row.run_timestamp?.split('T')[0],
|
|
229
|
+
dataset: row.dataset_id,
|
|
230
|
+
// Contamination lane (FAIL SAFE) — relative_only rows are valid only
|
|
231
|
+
// for comparing methods on the same corpus, NOT as absolute quality.
|
|
232
|
+
contamination: rowContamination(row),
|
|
233
|
+
relative_only: rowIsRelativeOnly(row),
|
|
234
|
+
score_lane: rowIsRelativeOnly(row) ? LANE_RELATIVE_ONLY : LANE_ABSOLUTE,
|
|
235
|
+
}));
|
|
236
|
+
}
|
|
237
|
+
return 0;
|
|
238
|
+
}
|
|
239
|
+
|
|
240
|
+
// ---- Human-readable table output ----
|
|
241
|
+
|
|
242
|
+
const pairDisplay = pair ? formatPairDisplay(pair) : 'All Pairs';
|
|
243
|
+
const sortLabel = SORT_KEYS[sortKey].label;
|
|
244
|
+
|
|
245
|
+
output.raw('');
|
|
246
|
+
output.raw(` champollion — Method Leaderboard`);
|
|
247
|
+
output.raw(` ${pairDisplay} | Sorted by: ${sortLabel}`);
|
|
248
|
+
if (topN) output.raw(` Showing top ${topN} of ${rows.length}`);
|
|
249
|
+
output.raw('');
|
|
250
|
+
|
|
251
|
+
// Table header
|
|
252
|
+
const header = [
|
|
253
|
+
pad('#', 4),
|
|
254
|
+
pad('Model', 28),
|
|
255
|
+
pad('Condition', 12),
|
|
256
|
+
pad('Score', 7, 'right'),
|
|
257
|
+
pad('chrF++', 8, 'right'),
|
|
258
|
+
pad('EM%', 7, 'right'),
|
|
259
|
+
pad('Tier', 14),
|
|
260
|
+
pad('Date', 12),
|
|
261
|
+
pad('Lane', 9),
|
|
262
|
+
].join(' ');
|
|
263
|
+
|
|
264
|
+
output.raw(` ${header}`);
|
|
265
|
+
output.raw(` ${'─'.repeat(header.length)}`);
|
|
266
|
+
|
|
267
|
+
for (let i = 0; i < display.length; i++) {
|
|
268
|
+
const row = display[i];
|
|
269
|
+
const tier = TIER_LABELS[row.quality_tier?.toLowerCase()] || row.quality_tier || '—';
|
|
270
|
+
const date = row.run_timestamp?.split('T')[0] || '—';
|
|
271
|
+
|
|
272
|
+
const lane = rowIsRelativeOnly(row) ? 'rel-only' : 'abs';
|
|
273
|
+
const line = [
|
|
274
|
+
pad(String(i + 1), 4),
|
|
275
|
+
pad(row.model_slug || '—', 28),
|
|
276
|
+
pad(row.condition || '—', 12),
|
|
277
|
+
pad(fmtMetric(row.composite_score, 4), 7, 'right'),
|
|
278
|
+
pad(fmtMetric(row.chrf_plus_plus), 8, 'right'),
|
|
279
|
+
pad(fmtMetric(row.exact_match_rate), 7, 'right'),
|
|
280
|
+
pad(tier, 14),
|
|
281
|
+
pad(date, 12),
|
|
282
|
+
pad(lane, 9),
|
|
283
|
+
].join(' ');
|
|
284
|
+
|
|
285
|
+
output.raw(` ${line}`);
|
|
286
|
+
}
|
|
287
|
+
|
|
288
|
+
output.raw('');
|
|
289
|
+
output.raw(` ${display.length} result${display.length !== 1 ? 's' : ''} shown.`);
|
|
290
|
+
output.raw(` Lane: abs = absolute-quality · rel-only = relative-comparison-only`);
|
|
291
|
+
output.raw(` (HIGH/MEDIUM contamination, FLORES, or unknown grade — compare within that corpus only)`);
|
|
292
|
+
output.raw(` → Install a method: champollion leaderboard --install <rank>`);
|
|
293
|
+
output.raw(` View full leaderboard: https://champollion.dev/leaderboard`);
|
|
294
|
+
output.raw('');
|
|
295
|
+
|
|
296
|
+
} catch (err) {
|
|
297
|
+
output.error(`Failed to fetch leaderboard: ${err.message}`);
|
|
298
|
+
return 1;
|
|
299
|
+
}
|
|
300
|
+
|
|
301
|
+
return 0;
|
|
302
|
+
}
|
|
303
|
+
|
|
304
|
+
// ---------------------------------------------------------------------------
|
|
305
|
+
// --install implementation
|
|
306
|
+
// ---------------------------------------------------------------------------
|
|
307
|
+
|
|
308
|
+
/**
|
|
309
|
+
* Extract method configuration from a leaderboard entry and write it as a
|
|
310
|
+
* champollion method plugin manifest.
|
|
311
|
+
*
|
|
312
|
+
* The generated manifest can be used with `champollion sync` by setting
|
|
313
|
+
* `"methodPlugin": "<name>"` in the pair config. The manifest follows
|
|
314
|
+
* the champollion-plugin.schema.json format.
|
|
315
|
+
*
|
|
316
|
+
* @param {Array} rows - Sorted leaderboard rows from Supabase
|
|
317
|
+
* @param {number} rank - 1-based rank to install
|
|
318
|
+
* @param {string} sortKey - Active sort key (for display context)
|
|
319
|
+
* @param {string} cwd - Current working directory
|
|
320
|
+
* @returns {Promise<number>} Exit code
|
|
321
|
+
*/
|
|
322
|
+
async function _handleInstall(rows, rank, sortKey, cwd, args = {}) {
|
|
323
|
+
if (rank < 1 || rank > rows.length) {
|
|
324
|
+
output.error(`Rank ${rank} is out of range (1–${rows.length}).`);
|
|
325
|
+
return 1;
|
|
326
|
+
}
|
|
327
|
+
|
|
328
|
+
const row = rows[rank - 1];
|
|
329
|
+
const runCard = row.run_card || {};
|
|
330
|
+
const methodCard = runCard.method_card || null;
|
|
331
|
+
const condition = row.condition || 'unknown';
|
|
332
|
+
const model = row.model_slug || 'unknown';
|
|
333
|
+
const pair = row.language_pair || '';
|
|
334
|
+
const [, targetLang] = pair.includes('>') ? pair.split('>') : ['', ''];
|
|
335
|
+
|
|
336
|
+
// Generate a plugin name from the method details
|
|
337
|
+
const safeName = [
|
|
338
|
+
targetLang || 'any',
|
|
339
|
+
condition,
|
|
340
|
+
model.replace(/[^a-z0-9]/gi, '-').toLowerCase(),
|
|
341
|
+
].filter(Boolean).join('-').replace(/--+/g, '-');
|
|
342
|
+
|
|
343
|
+
// Use the canonical MethodConfig from the harness run card if available.
|
|
344
|
+
// Fall back to reconstruction from individual fields for backward compat
|
|
345
|
+
// with old run cards that don't include method_config.
|
|
346
|
+
const methodConfig = runCard.method_config || {
|
|
347
|
+
model: model,
|
|
348
|
+
temperature: runCard.temperature ?? 0.3,
|
|
349
|
+
batchSize: runCard.batch_size ?? 80,
|
|
350
|
+
register: null,
|
|
351
|
+
coachingFile: null,
|
|
352
|
+
coachingPrompt: null,
|
|
353
|
+
promptContext: null,
|
|
354
|
+
qualityTier: null,
|
|
355
|
+
};
|
|
356
|
+
|
|
357
|
+
// Build the plugin manifest (champollion-plugin.schema.json format)
|
|
358
|
+
const manifest = {
|
|
359
|
+
name: safeName,
|
|
360
|
+
type: _inferMethodType(condition, runCard),
|
|
361
|
+
version: '1.0.0',
|
|
362
|
+
description: methodCard?.description
|
|
363
|
+
|| `Method extracted from leaderboard rank #${rank} (${formatPairDisplay(pair)}, ${condition}).`,
|
|
364
|
+
author: row.submitter || methodCard?.author || 'unknown',
|
|
365
|
+
locales: targetLang ? [targetLang] : ['unknown'], // schema requires minItems: 1
|
|
366
|
+
config: methodConfig,
|
|
367
|
+
benchmarks: {
|
|
368
|
+
[targetLang || 'unknown']: {
|
|
369
|
+
date: row.run_timestamp || new Date().toISOString(),
|
|
370
|
+
corpus_size: row.corpus_size || runCard.dataset?.entry_count || 0,
|
|
371
|
+
exact_match_rate: row.exact_match_rate ?? 0,
|
|
372
|
+
corpus_chrf: row.chrf_plus_plus ?? 0,
|
|
373
|
+
model: model,
|
|
374
|
+
harness_version: row.harness_version || runCard.harness_version || '',
|
|
375
|
+
},
|
|
376
|
+
},
|
|
377
|
+
provenance: {
|
|
378
|
+
resources: [],
|
|
379
|
+
commercialReady: false,
|
|
380
|
+
flags: ['extracted-from-leaderboard'],
|
|
381
|
+
},
|
|
382
|
+
};
|
|
383
|
+
|
|
384
|
+
// Note: method_card data is available in the Supabase run_card but is NOT
|
|
385
|
+
// included here — the plugin manifest follows champollion-plugin.schema.json
|
|
386
|
+
// which does not allow _method_card (additionalProperties: false).
|
|
387
|
+
|
|
388
|
+
// If condition is coached, note that coaching data is needed
|
|
389
|
+
if (condition.includes('coached') || _inferMethodType(condition, runCard) === 'llm-coached') {
|
|
390
|
+
manifest.coaching = { dir: 'coaching' };
|
|
391
|
+
}
|
|
392
|
+
|
|
393
|
+
// Write to .champollion/methods/<name>/method.json (directory convention per plugins.js)
|
|
394
|
+
const pluginDir = path.join(cwd, '.champollion', 'methods', safeName);
|
|
395
|
+
const outPath = path.join(pluginDir, 'method.json');
|
|
396
|
+
|
|
397
|
+
fs.mkdirSync(pluginDir, { recursive: true });
|
|
398
|
+
fs.writeFileSync(outPath, JSON.stringify(manifest, null, 2) + '\n', 'utf-8');
|
|
399
|
+
|
|
400
|
+
// Display confirmation
|
|
401
|
+
output.raw('');
|
|
402
|
+
output.ok(`Installed method: ${safeName}`);
|
|
403
|
+
output.raw(` → ${outPath}`);
|
|
404
|
+
output.raw('');
|
|
405
|
+
output.raw(` Source: Rank #${rank} on leaderboard (${formatPairDisplay(pair)})`);
|
|
406
|
+
output.raw(` Model: ${model}`);
|
|
407
|
+
output.raw(` Score: ${fmtMetric(row.composite_score, 4)} composite (${row.quality_tier || 'unscored'})`);
|
|
408
|
+
if (methodCard?.name) {
|
|
409
|
+
output.raw(` Method: ${methodCard.name} (${methodCard.class || 'unclassified'})`);
|
|
410
|
+
}
|
|
411
|
+
output.raw('');
|
|
412
|
+
|
|
413
|
+
// --apply: auto-wire the installed plugin into champollion.config.json
|
|
414
|
+
if (args.apply) {
|
|
415
|
+
const configPath = path.join(cwd, 'champollion.config.json');
|
|
416
|
+
if (fs.existsSync(configPath)) {
|
|
417
|
+
try {
|
|
418
|
+
const configRaw = fs.readFileSync(configPath, 'utf-8');
|
|
419
|
+
const config = JSON.parse(configRaw);
|
|
420
|
+
|
|
421
|
+
// Ensure pairs object exists
|
|
422
|
+
if (!config.pairs) config.pairs = {};
|
|
423
|
+
|
|
424
|
+
// Build the pair key from the leaderboard entry
|
|
425
|
+
const [srcLang, tgtLang] = pair.includes('>') ? pair.split('>') : ['en', targetLang || 'unknown'];
|
|
426
|
+
const pairKey = `${srcLang}:${tgtLang}`;
|
|
427
|
+
|
|
428
|
+
// Set or update the pair entry with the plugin reference
|
|
429
|
+
config.pairs[pairKey] = config.pairs[pairKey] || {};
|
|
430
|
+
config.pairs[pairKey].methodPlugin = safeName;
|
|
431
|
+
|
|
432
|
+
fs.writeFileSync(configPath, JSON.stringify(config, null, 2) + '\n', 'utf-8');
|
|
433
|
+
output.ok(`Applied plugin to config: pairs["${pairKey}"].methodPlugin = "${safeName}"`);
|
|
434
|
+
output.raw(` → ${configPath}`);
|
|
435
|
+
} catch (err) {
|
|
436
|
+
output.error(`Failed to update config: ${err.message}`);
|
|
437
|
+
output.raw(' You can manually add the plugin reference to your config.');
|
|
438
|
+
}
|
|
439
|
+
} else {
|
|
440
|
+
output.raw(' [WARN] No champollion.config.json found — skipping --apply.');
|
|
441
|
+
output.raw(' Run `champollion init` first, then re-install with --apply.');
|
|
442
|
+
}
|
|
443
|
+
output.raw('');
|
|
444
|
+
} else {
|
|
445
|
+
output.raw(' To use this method in your project:');
|
|
446
|
+
output.raw(` "methodPlugin": "${safeName}"`);
|
|
447
|
+
output.raw('');
|
|
448
|
+
output.raw(' Or auto-wire it: champollion leaderboard --install ' + rank + ' --apply');
|
|
449
|
+
output.raw('');
|
|
450
|
+
}
|
|
451
|
+
|
|
452
|
+
if (manifest.coaching) {
|
|
453
|
+
output.raw(' ⚠ This method uses coaching data. Create a coaching/ directory');
|
|
454
|
+
output.raw(' in the plugin folder with language-specific examples.');
|
|
455
|
+
output.raw('');
|
|
456
|
+
}
|
|
457
|
+
|
|
458
|
+
return 0;
|
|
459
|
+
}
|
|
460
|
+
|
|
461
|
+
/**
|
|
462
|
+
* Infer the champollion method type from a condition string and run card.
|
|
463
|
+
* Maps arena condition names to plugin schema type enum values.
|
|
464
|
+
*/
|
|
465
|
+
function _inferMethodType(condition, runCard) {
|
|
466
|
+
if (!condition) return 'llm';
|
|
467
|
+
const c = condition.toLowerCase();
|
|
468
|
+
if (c.includes('coached')) return 'llm-coached';
|
|
469
|
+
if (c.includes('fst')) return 'llm-coached'; // FST-gated methods are coached pipelines
|
|
470
|
+
if (c.includes('google')) return 'google-translate';
|
|
471
|
+
if (c.includes('deepl')) return 'deepl';
|
|
472
|
+
if (c.includes('microsoft')) return 'microsoft-translator';
|
|
473
|
+
if (runCard?.tools_enabled) return 'llm';
|
|
474
|
+
return 'llm';
|
|
475
|
+
}
|
|
476
|
+
|
|
477
|
+
export { run };
|
|
478
|
+
|
|
@@ -0,0 +1,30 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Command: lint
|
|
3
|
+
*
|
|
4
|
+
* Scans source files for hardcoded user-facing strings that should
|
|
5
|
+
* be wrapped in t() calls. Returns exit code 1 if issues found
|
|
6
|
+
* (unless --warn-only is set).
|
|
7
|
+
*/
|
|
8
|
+
|
|
9
|
+
import { runLint } from '../lint.js';
|
|
10
|
+
import { output } from '../output.js';
|
|
11
|
+
|
|
12
|
+
/**
|
|
13
|
+
* @param {import('../types.js').CLIArgs} args - Parsed CLI arguments
|
|
14
|
+
* @param {string} cwd - Working directory
|
|
15
|
+
* @returns {Promise<number>} Exit code (0 = success, 1 = error)
|
|
16
|
+
*/
|
|
17
|
+
async function run(args, cwd) {
|
|
18
|
+
// --json: stdout carries exactly one JSON document (printed by runLint).
|
|
19
|
+
// Quiet mode keeps the human progress/report lines off stdout.
|
|
20
|
+
if (args.json) output.setMode('quiet');
|
|
21
|
+
|
|
22
|
+
const exitCode = await runLint({
|
|
23
|
+
cwd,
|
|
24
|
+
cliArgs: args,
|
|
25
|
+
warnOnly: !!args['warn-only'],
|
|
26
|
+
});
|
|
27
|
+
return exitCode;
|
|
28
|
+
}
|
|
29
|
+
|
|
30
|
+
export { run };
|