@kevinrabun/judges 3.119.0 → 3.121.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +1 -1
- package/dist/api.d.ts +2 -1
- package/dist/api.js +3 -1
- package/dist/cli-dispatch.d.ts +7 -0
- package/dist/cli-dispatch.js +654 -0
- package/dist/cli-formatters.d.ts +6 -0
- package/dist/cli-formatters.js +186 -0
- package/dist/cli.js +69 -4159
- package/dist/commands/baseline.js +2 -42
- package/dist/commands/coverage.js +3 -39
- package/dist/commands/diff.js +2 -38
- package/dist/commands/fix-pr.js +2 -23
- package/dist/commands/fix.js +3 -27
- package/dist/commands/llm-benchmark.d.ts +7 -0
- package/dist/commands/llm-benchmark.js +27 -1
- package/dist/commands/quality-gate.js +1 -12
- package/dist/commands/review-parallel.js +1 -19
- package/dist/commands/review.js +2 -33
- package/dist/commands/rule-test.js +1 -15
- package/dist/commands/tune.js +2 -29
- package/dist/commands/watch.js +3 -42
- package/dist/evaluators/hallucination-detection.js +343 -0
- package/dist/evaluators/index.d.ts +2 -11
- package/dist/evaluators/index.js +3 -181
- package/dist/evaluators/security.js +226 -2
- package/dist/evaluators/suppressions.d.ts +49 -0
- package/dist/evaluators/suppressions.js +185 -0
- package/dist/ext-to-lang.d.ts +16 -0
- package/dist/ext-to-lang.js +60 -0
- package/dist/github-app.d.ts +1 -3
- package/dist/github-app.js +2 -34
- package/dist/parallel.js +2 -14
- package/dist/tools/register-evaluation.js +2 -29
- package/package.json +1 -1
- package/server.json +2 -2
|
@@ -484,6 +484,349 @@ const HALLUCINATED_PATTERNS = [
|
|
|
484
484
|
fix: "Verify the package exists on Maven Central/Gradle Plugin Portal. Use established alternatives from Spring Security, Apache Commons, or Bouncy Castle.",
|
|
485
485
|
languages: ["java", "kotlin"],
|
|
486
486
|
},
|
|
487
|
+
// ── Additional Node.js / JavaScript / TypeScript ──────────────────────
|
|
488
|
+
// crypto.hash() doesn't exist — it's crypto.createHash()
|
|
489
|
+
{
|
|
490
|
+
pattern: /\bcrypto\.hash\s*\(/,
|
|
491
|
+
hallucinated: "crypto.hash()",
|
|
492
|
+
reason: "Node.js crypto module has no hash() method. LLMs hallucinate a simplified API.",
|
|
493
|
+
fix: "Use crypto.createHash('sha256').update(data).digest('hex').",
|
|
494
|
+
languages: ["javascript", "typescript"],
|
|
495
|
+
},
|
|
496
|
+
// Promise.map/filter/timeout/retry/sequential — don't exist on native Promise
|
|
497
|
+
{
|
|
498
|
+
pattern: /\bPromise\.(?:map|filter|timeout|retry|sequential)\s*\(/,
|
|
499
|
+
hallucinated: "Promise.map/filter/timeout/retry/sequential()",
|
|
500
|
+
reason: "Native Promise has no map(), filter(), timeout(), retry(), or sequential() methods. LLMs hallucinate these from Bluebird or other promise libraries.",
|
|
501
|
+
fix: "Use Promise.all() with Array.map() for parallel, or implement custom retry/timeout logic.",
|
|
502
|
+
languages: ["javascript", "typescript"],
|
|
503
|
+
},
|
|
504
|
+
// Fake Node.js built-in submodules (node:url/validator, node:path/sanitize, etc.)
|
|
505
|
+
{
|
|
506
|
+
pattern: /\bfrom\s+['"]node:(?:url|path|net|tls|timers|util|worker_threads)\/\w+['"]/,
|
|
507
|
+
hallucinated: "Non-existent Node.js built-in submodule",
|
|
508
|
+
reason: "Node.js built-in modules do not have these submodule paths. LLMs fabricate submodule paths by combining real module names with plausible feature names.",
|
|
509
|
+
fix: "Import directly from the parent module (e.g., import { URL } from 'node:url').",
|
|
510
|
+
languages: ["javascript", "typescript"],
|
|
511
|
+
},
|
|
512
|
+
// os.getCpuUsagePercent / os.getMemoryUsagePercent / os.getDiskUsagePercent
|
|
513
|
+
{
|
|
514
|
+
pattern: /\bos\.(?:getCpuUsagePercent|getMemoryUsagePercent|getDiskUsagePercent)\s*\(/,
|
|
515
|
+
hallucinated: "os.getCpuUsagePercent/getMemoryUsagePercent/getDiskUsagePercent()",
|
|
516
|
+
reason: "Node.js os module has no percentage-based resource usage methods. LLMs fabricate convenient but non-existent APIs.",
|
|
517
|
+
fix: "Use os.cpus() for CPU info, os.freemem()/os.totalmem() for memory.",
|
|
518
|
+
languages: ["javascript", "typescript"],
|
|
519
|
+
},
|
|
520
|
+
// process.onUncaughtRejection / process.setMaxMemory / process.enableGracefulShutdown
|
|
521
|
+
{
|
|
522
|
+
pattern: /\bprocess\.(?:onUncaughtRejection|setMaxMemory|enableGracefulShutdown)\s*\(/,
|
|
523
|
+
hallucinated: "process.onUncaughtRejection/setMaxMemory/enableGracefulShutdown()",
|
|
524
|
+
reason: "Node.js process object does not have these methods. LLMs fabricate convenience APIs.",
|
|
525
|
+
fix: "Use process.on('unhandledRejection', handler). Use --max-old-space-size for memory. Implement graceful shutdown with process.on('SIGTERM').",
|
|
526
|
+
languages: ["javascript", "typescript"],
|
|
527
|
+
},
|
|
528
|
+
// Fake TypeScript utility types presented as built-in
|
|
529
|
+
{
|
|
530
|
+
pattern: /\b(?:StrictOmit|Validated|Frozen)\s*</,
|
|
531
|
+
hallucinated: "Non-existent TypeScript built-in utility type",
|
|
532
|
+
reason: "TypeScript does not have built-in StrictOmit, Validated, or Frozen utility types. LLMs hallucinate these as part of the standard type system.",
|
|
533
|
+
fix: "Use built-in types: Omit<T, K> for StrictOmit, Readonly<T> for Frozen. Define custom types for other needs.",
|
|
534
|
+
languages: ["typescript"],
|
|
535
|
+
},
|
|
536
|
+
// ── Additional Python ─────────────────────────────────────────────────
|
|
537
|
+
// requests.async_get/post/etc. — requests has no async methods
|
|
538
|
+
{
|
|
539
|
+
pattern: /\brequests\.async_(?:get|post|put|delete|patch)\s*\(/,
|
|
540
|
+
hallucinated: "requests.async_get()",
|
|
541
|
+
reason: "The requests library has no async methods. LLMs hallucinate async variants of the synchronous API.",
|
|
542
|
+
fix: "Use aiohttp or httpx for async HTTP: async with aiohttp.ClientSession() as s: await s.get(url).",
|
|
543
|
+
languages: ["python"],
|
|
544
|
+
},
|
|
545
|
+
// os.makedirs with permissions= parameter (should be mode=)
|
|
546
|
+
{
|
|
547
|
+
pattern: /\bos\.makedirs\s*\([^)]*\bpermissions\s*=/,
|
|
548
|
+
hallucinated: "os.makedirs(permissions=...)",
|
|
549
|
+
reason: "os.makedirs() uses 'mode=' for permissions, not 'permissions='. LLMs hallucinate a more readable parameter name.",
|
|
550
|
+
fix: "Use os.makedirs(path, mode=0o755, exist_ok=True).",
|
|
551
|
+
languages: ["python"],
|
|
552
|
+
},
|
|
553
|
+
// collections.OrderedDefaultDict doesn't exist
|
|
554
|
+
{
|
|
555
|
+
pattern: /\bfrom\s+collections\s+import\b.*\bOrderedDefaultDict\b/,
|
|
556
|
+
hallucinated: "collections.OrderedDefaultDict",
|
|
557
|
+
reason: "Python's collections module has no OrderedDefaultDict. LLMs fabricate this by combining OrderedDict and defaultdict.",
|
|
558
|
+
fix: "Use collections.OrderedDict or collections.defaultdict separately.",
|
|
559
|
+
languages: ["python"],
|
|
560
|
+
},
|
|
561
|
+
// typing.StrictDict doesn't exist
|
|
562
|
+
{
|
|
563
|
+
pattern: /\bfrom\s+typing\s+import\b.*\bStrictDict\b/,
|
|
564
|
+
hallucinated: "typing.StrictDict",
|
|
565
|
+
reason: "Python's typing module has no StrictDict. LLMs fabricate convenience types.",
|
|
566
|
+
fix: "Use typing.TypedDict for typed dicts or typing.Dict for generic dict hints.",
|
|
567
|
+
languages: ["python"],
|
|
568
|
+
},
|
|
569
|
+
// pathlib.SecurePath doesn't exist
|
|
570
|
+
{
|
|
571
|
+
pattern: /\bfrom\s+pathlib\s+import\b.*\bSecurePath\b/,
|
|
572
|
+
hallucinated: "pathlib.SecurePath",
|
|
573
|
+
reason: "Python's pathlib has no SecurePath class. LLMs fabricate security-focused variants.",
|
|
574
|
+
fix: "Use pathlib.Path and validate/sanitize paths manually.",
|
|
575
|
+
languages: ["python"],
|
|
576
|
+
},
|
|
577
|
+
// asyncio.ParallelMap doesn't exist
|
|
578
|
+
{
|
|
579
|
+
pattern: /\bfrom\s+asyncio\s+import\b.*\bParallelMap\b/,
|
|
580
|
+
hallucinated: "asyncio.ParallelMap",
|
|
581
|
+
reason: "Python's asyncio has no ParallelMap. LLMs fabricate parallel execution utilities.",
|
|
582
|
+
fix: "Use asyncio.gather(*[coro(x) for x in items]).",
|
|
583
|
+
languages: ["python"],
|
|
584
|
+
},
|
|
585
|
+
// json.schema doesn't exist in Python stdlib
|
|
586
|
+
{
|
|
587
|
+
pattern: /\bjson\.schema\b/,
|
|
588
|
+
hallucinated: "json.schema",
|
|
589
|
+
reason: "Python's json module has no schema submodule. LLMs conflate json with the jsonschema package.",
|
|
590
|
+
fix: "Install jsonschema: from jsonschema import validate.",
|
|
591
|
+
languages: ["python"],
|
|
592
|
+
},
|
|
593
|
+
// functools.memoize doesn't exist (it's lru_cache or cache)
|
|
594
|
+
{
|
|
595
|
+
pattern: /\bfrom\s+functools\s+import\b.*\bmemoize\b/,
|
|
596
|
+
hallucinated: "functools.memoize",
|
|
597
|
+
reason: "Python's functools has no memoize. LLMs hallucinate this from other languages.",
|
|
598
|
+
fix: "Use @functools.lru_cache(maxsize=128) or @functools.cache (Python 3.9+).",
|
|
599
|
+
languages: ["python"],
|
|
600
|
+
},
|
|
601
|
+
// ── Additional Java ───────────────────────────────────────────────────
|
|
602
|
+
// stream().filterMap() doesn't exist in Java (Rust concept)
|
|
603
|
+
{
|
|
604
|
+
pattern: /\.filterMap\s*\(/,
|
|
605
|
+
hallucinated: ".filterMap()",
|
|
606
|
+
reason: "Java Streams have no filterMap(). LLMs hallucinate this from Rust's filter_map().",
|
|
607
|
+
fix: "Use .filter(predicate).map(mapper) as two separate operations.",
|
|
608
|
+
languages: ["java"],
|
|
609
|
+
scopeCheckMethod: "filterMap",
|
|
610
|
+
},
|
|
611
|
+
// Stream.ofParallel() doesn't exist
|
|
612
|
+
{
|
|
613
|
+
pattern: /\bStream\.ofParallel\s*\(/,
|
|
614
|
+
hallucinated: "Stream.ofParallel()",
|
|
615
|
+
reason: "Java has no Stream.ofParallel(). LLMs fabricate this combining Stream.of() and parallelStream().",
|
|
616
|
+
fix: "Use collection.parallelStream() or Stream.of(...).parallel().",
|
|
617
|
+
languages: ["java"],
|
|
618
|
+
},
|
|
619
|
+
// Stream.zip() doesn't exist in Java stdlib
|
|
620
|
+
{
|
|
621
|
+
pattern: /\bStream\.zip\s*\(/,
|
|
622
|
+
hallucinated: "Stream.zip()",
|
|
623
|
+
reason: "Java Streams have no zip(). LLMs hallucinate this from Scala, Kotlin, or Python.",
|
|
624
|
+
fix: "Use IntStream.range() for manual zipping, or Guava's Streams.zip().",
|
|
625
|
+
languages: ["java"],
|
|
626
|
+
},
|
|
627
|
+
// .filterAsync() doesn't exist in Java Streams or C# LINQ
|
|
628
|
+
{
|
|
629
|
+
pattern: /\.filterAsync\s*\(/,
|
|
630
|
+
hallucinated: ".filterAsync()",
|
|
631
|
+
reason: "Neither Java Streams nor C# LINQ have filterAsync(). LLMs fabricate async variants.",
|
|
632
|
+
fix: "Use CompletableFuture with .filter() in Java, or async/await with Where() in C#.",
|
|
633
|
+
languages: ["java", "csharp"],
|
|
634
|
+
scopeCheckMethod: "filterAsync",
|
|
635
|
+
},
|
|
636
|
+
// Collectors.toUnmodifiableGroupingBy doesn't exist
|
|
637
|
+
{
|
|
638
|
+
pattern: /\bCollectors\.toUnmodifiableGroupingBy\s*\(/,
|
|
639
|
+
hallucinated: "Collectors.toUnmodifiableGroupingBy()",
|
|
640
|
+
reason: "Java has no Collectors.toUnmodifiableGroupingBy(). LLMs combine groupingBy() with unmodifiable concepts.",
|
|
641
|
+
fix: "Use Collectors.groupingBy() and wrap with Collections.unmodifiableMap().",
|
|
642
|
+
languages: ["java"],
|
|
643
|
+
},
|
|
644
|
+
// .groupByKey() on Java streams (Spark/Kotlin concept)
|
|
645
|
+
{
|
|
646
|
+
pattern: /\.groupByKey\s*\(/,
|
|
647
|
+
hallucinated: ".groupByKey()",
|
|
648
|
+
reason: "Java Streams have no groupByKey(). LLMs hallucinate this from Spark or Kotlin.",
|
|
649
|
+
fix: "Use .collect(Collectors.groupingBy(keyFunction)).",
|
|
650
|
+
languages: ["java"],
|
|
651
|
+
scopeCheckMethod: "groupByKey",
|
|
652
|
+
},
|
|
653
|
+
// .toConcurrentMap() terminal operation doesn't exist
|
|
654
|
+
{
|
|
655
|
+
pattern: /\.toConcurrentMap\s*\(\s*\)/,
|
|
656
|
+
hallucinated: ".toConcurrentMap()",
|
|
657
|
+
reason: "Java Streams have no .toConcurrentMap() terminal operation.",
|
|
658
|
+
fix: "Use .collect(Collectors.toConcurrentMap(keyMapper, valueMapper)).",
|
|
659
|
+
languages: ["java"],
|
|
660
|
+
scopeCheckMethod: "toConcurrentMap",
|
|
661
|
+
},
|
|
662
|
+
// ── Additional C# ────────────────────────────────────────────────────
|
|
663
|
+
// Fake LINQ extension methods
|
|
664
|
+
{
|
|
665
|
+
pattern: /\.(?:WhereAsync|BatchBy|ParallelSelect|FlattenAll|SortByMultiple|TakeWhileIncluding|SlidingWindow)\s*\(/,
|
|
666
|
+
hallucinated: "Non-existent LINQ extension method",
|
|
667
|
+
reason: "C# LINQ does not have WhereAsync, BatchBy, ParallelSelect, FlattenAll, SortByMultiple, TakeWhileIncluding, or SlidingWindow. LLMs hallucinate these extensions.",
|
|
668
|
+
fix: "Use standard LINQ: Where, Chunk (.NET 6+), AsParallel().Select, SelectMany, OrderBy.ThenBy, TakeWhile.",
|
|
669
|
+
languages: ["csharp"],
|
|
670
|
+
},
|
|
671
|
+
// ── Additional Go ─────────────────────────────────────────────────────
|
|
672
|
+
// 'implements' keyword in Go type constraints (Java/C# concept)
|
|
673
|
+
{
|
|
674
|
+
pattern: /\bimplements\s+\w+/,
|
|
675
|
+
hallucinated: "'implements' keyword in Go generics",
|
|
676
|
+
reason: "Go does not have an 'implements' keyword. LLMs hallucinate this from Java/C#.",
|
|
677
|
+
fix: "Use Go type constraints: [T comparable], [T constraints.Ordered], or define a constraint interface.",
|
|
678
|
+
languages: ["go"],
|
|
679
|
+
},
|
|
680
|
+
// ── Additional Patterns (Benchmark Gaps) ──────────────────────────────
|
|
681
|
+
// String.contains() doesn't exist in JS — Java confusion
|
|
682
|
+
{
|
|
683
|
+
pattern: /\.\bcontains\s*\(\s*(?:['"`]|[a-zA-Z_])/,
|
|
684
|
+
hallucinated: "String.contains()",
|
|
685
|
+
reason: "JavaScript strings have no .contains() method. This is a Java API hallucinated onto JS.",
|
|
686
|
+
fix: "Use .includes() instead of .contains().",
|
|
687
|
+
languages: ["javascript"],
|
|
688
|
+
scopeCheckMethod: "contains",
|
|
689
|
+
},
|
|
690
|
+
// crypto.signMessage() doesn't exist in Node.js
|
|
691
|
+
{
|
|
692
|
+
pattern: /\bcrypto\.signMessage\s*\(/,
|
|
693
|
+
hallucinated: "crypto.signMessage()",
|
|
694
|
+
reason: "Node.js crypto module has no signMessage() method.",
|
|
695
|
+
fix: "Use crypto.sign() or crypto.createSign() to sign data.",
|
|
696
|
+
languages: ["javascript", "typescript"],
|
|
697
|
+
},
|
|
698
|
+
// fetch().abort() — fetch returns a Promise, not an abortable request
|
|
699
|
+
{
|
|
700
|
+
pattern: /\.abort\s*\(\s*\)/,
|
|
701
|
+
hallucinated: "fetch().abort()",
|
|
702
|
+
reason: "fetch() returns a Promise, not an abortable request. There is no .abort() method on the returned value.",
|
|
703
|
+
fix: "Use AbortController: const controller = new AbortController(); fetch(url, { signal: controller.signal }); controller.abort();",
|
|
704
|
+
languages: ["javascript", "typescript"],
|
|
705
|
+
scopeCheckMethod: "abort",
|
|
706
|
+
requiresImport: "fetch",
|
|
707
|
+
},
|
|
708
|
+
// Python typing.Protocol.implements() doesn't exist
|
|
709
|
+
{
|
|
710
|
+
pattern: /\bProtocol\.implements\s*\(/,
|
|
711
|
+
hallucinated: "Protocol.implements()",
|
|
712
|
+
reason: "Python's typing.Protocol has no implements() class method. Protocol checking is structural.",
|
|
713
|
+
fix: "Use isinstance() with @runtime_checkable decorator, or rely on mypy/pyright for static checks.",
|
|
714
|
+
languages: ["python"],
|
|
715
|
+
},
|
|
716
|
+
// json.loads() on a file path — should be json.load(file_handle)
|
|
717
|
+
{
|
|
718
|
+
pattern: /\bjson\.loads\s*\(\s*(?:path|file_?path|config_?path|filename)\b/,
|
|
719
|
+
hallucinated: "json.loads(file_path)",
|
|
720
|
+
reason: "json.loads() parses a string, not a file path. LLMs confuse json.loads() with json.load().",
|
|
721
|
+
fix: "Use json.load(open(path)) or: with open(path) as f: data = json.load(f)",
|
|
722
|
+
languages: ["python"],
|
|
723
|
+
},
|
|
724
|
+
// Prisma fabricated methods
|
|
725
|
+
{
|
|
726
|
+
pattern: /\.groupByAndCount\s*\(/,
|
|
727
|
+
hallucinated: "prisma.model.groupByAndCount()",
|
|
728
|
+
reason: "Prisma has no groupByAndCount(). Use groupBy() with _count aggregation.",
|
|
729
|
+
fix: "Use prisma.model.groupBy({ by: ['field'], _count: true })",
|
|
730
|
+
languages: ["javascript", "typescript"],
|
|
731
|
+
},
|
|
732
|
+
{
|
|
733
|
+
pattern: /\.bulkUpsert\s*\(/,
|
|
734
|
+
hallucinated: "prisma.model.bulkUpsert()",
|
|
735
|
+
reason: "Prisma has no bulkUpsert(). Use createMany() or loop with upsert().",
|
|
736
|
+
fix: "Use prisma.model.createMany({ data: items }) or loop with prisma.model.upsert()",
|
|
737
|
+
languages: ["javascript", "typescript"],
|
|
738
|
+
},
|
|
739
|
+
{
|
|
740
|
+
pattern: /\.findManyOrThrow\s*\(/,
|
|
741
|
+
hallucinated: "prisma.model.findManyOrThrow()",
|
|
742
|
+
reason: "Prisma has findFirstOrThrow() and findUniqueOrThrow() but not findManyOrThrow().",
|
|
743
|
+
fix: "Use findMany() and check result length, or findFirstOrThrow() for single records.",
|
|
744
|
+
languages: ["javascript", "typescript"],
|
|
745
|
+
},
|
|
746
|
+
{
|
|
747
|
+
pattern: /\.softDelete\s*\(/,
|
|
748
|
+
hallucinated: "prisma.model.softDelete()",
|
|
749
|
+
reason: "Prisma has no built-in softDelete() method.",
|
|
750
|
+
fix: "Implement soft delete: prisma.model.update({ where: { id }, data: { deletedAt: new Date() } })",
|
|
751
|
+
languages: ["javascript", "typescript"],
|
|
752
|
+
},
|
|
753
|
+
// Rust std hallucinations
|
|
754
|
+
{
|
|
755
|
+
pattern: /\bget_or_default\s*\(/,
|
|
756
|
+
hallucinated: "HashMap.get_or_default()",
|
|
757
|
+
reason: "Rust HashMap has no get_or_default(). Java HashMap API hallucinated onto Rust.",
|
|
758
|
+
fix: "Use .entry(key).or_default() or .get(key).unwrap_or(&default).",
|
|
759
|
+
languages: ["rust"],
|
|
760
|
+
},
|
|
761
|
+
{
|
|
762
|
+
pattern: /\bVec::from_iter_parallel\b/,
|
|
763
|
+
hallucinated: "Vec::from_iter_parallel()",
|
|
764
|
+
reason: "Rust Vec has no from_iter_parallel() method.",
|
|
765
|
+
fix: "Use rayon: (0..1000).into_par_iter().map(|x| x * 2).collect()",
|
|
766
|
+
languages: ["rust"],
|
|
767
|
+
},
|
|
768
|
+
{
|
|
769
|
+
pattern: /\bArc::try_make_mut\b/,
|
|
770
|
+
hallucinated: "Arc::try_make_mut()",
|
|
771
|
+
reason: "Rust Arc has no try_make_mut(). Arc::make_mut() exists but requires Clone.",
|
|
772
|
+
fix: "Use Arc::make_mut() (clones if needed) or Arc::try_unwrap().",
|
|
773
|
+
languages: ["rust"],
|
|
774
|
+
},
|
|
775
|
+
{
|
|
776
|
+
pattern: /\btruncate_safe\s*\(/,
|
|
777
|
+
hallucinated: "String::truncate_safe()",
|
|
778
|
+
reason: "Rust String has no truncate_safe() method.",
|
|
779
|
+
fix: "Use .truncate(n) with char boundary check: if s.is_char_boundary(n) { s.truncate(n); }",
|
|
780
|
+
languages: ["rust"],
|
|
781
|
+
},
|
|
782
|
+
// secure_random crate doesn't exist in Rust
|
|
783
|
+
{
|
|
784
|
+
pattern: /\buse\s+secure_random\b/,
|
|
785
|
+
hallucinated: "secure_random crate",
|
|
786
|
+
reason: "There is no 'secure_random' crate. This is a fabricated crate name.",
|
|
787
|
+
fix: "Use the 'rand' crate with OsRng or ThreadRng for cryptographic randomness.",
|
|
788
|
+
languages: ["rust"],
|
|
789
|
+
},
|
|
790
|
+
// Java stream().toArray(Constructor::new) — wrong signature
|
|
791
|
+
{
|
|
792
|
+
pattern: /\.stream\(\)\.toArray\s*\(\s*\w+::new\s*\)/,
|
|
793
|
+
hallucinated: "stream().toArray(Constructor::new)",
|
|
794
|
+
reason: "Stream.toArray() takes an IntFunction<A[]> (e.g., String[]::new), not a constructor reference.",
|
|
795
|
+
fix: "Use .toArray(String[]::new) for typed arrays, or .toArray() for Object[].",
|
|
796
|
+
languages: ["java"],
|
|
797
|
+
},
|
|
798
|
+
// Fabricated npm packages (common webpack/build tool hallucinations)
|
|
799
|
+
{
|
|
800
|
+
pattern: /\brequire\s*\(\s*["']webpack-(?:auto-optimize|security-scan|smart-split)["']\s*\)/,
|
|
801
|
+
hallucinated: "webpack-auto-optimize / webpack-security-scan / webpack-smart-split",
|
|
802
|
+
reason: "These webpack plugins do not exist. LLMs fabricate plausible-sounding plugin names.",
|
|
803
|
+
fix: "Use real webpack plugins: TerserPlugin, BundleAnalyzerPlugin, SplitChunksPlugin (built-in).",
|
|
804
|
+
languages: ["javascript", "typescript"],
|
|
805
|
+
},
|
|
806
|
+
// Fabricated AWS SDK commands
|
|
807
|
+
{
|
|
808
|
+
pattern: /\b(?:SecurityScanCommand|AutoScaleCommand|WarmUpCommand)\b/,
|
|
809
|
+
hallucinated: "Fabricated AWS SDK commands",
|
|
810
|
+
reason: "SecurityScanCommand, AutoScaleCommand, WarmUpCommand do not exist in the AWS SDK.",
|
|
811
|
+
fix: "Use real AWS SDK commands: ListObjectsV2Command, PutItemCommand, InvokeCommand, etc.",
|
|
812
|
+
languages: ["javascript", "typescript"],
|
|
813
|
+
},
|
|
814
|
+
// Fabricated Octokit/GitHub API methods
|
|
815
|
+
{
|
|
816
|
+
pattern: /\.repos\.(?:getSecurityScore|getAICodeReview|getPerformanceMetrics)\s*\(/,
|
|
817
|
+
hallucinated: "Fabricated Octokit methods",
|
|
818
|
+
reason: "Octokit has no getSecurityScore(), getAICodeReview(), or getPerformanceMetrics() methods.",
|
|
819
|
+
fix: "Use real GitHub API endpoints: repos.get(), repos.listCommits(), repos.getContent(), etc.",
|
|
820
|
+
languages: ["javascript", "typescript"],
|
|
821
|
+
},
|
|
822
|
+
// Fabricated SQL functions
|
|
823
|
+
{
|
|
824
|
+
pattern: /\b(?:TOP_N|STRING_AGG_DISTINCT|FIRST_VALUE_IF|WEIGHTED_AVG|RUNNING_TOTAL|AUTO_BUCKET|FUZZY_MATCH|FILL_GAPS)\s*\(/i,
|
|
825
|
+
hallucinated: "Fabricated SQL functions",
|
|
826
|
+
reason: "TOP_N, STRING_AGG_DISTINCT, WEIGHTED_AVG, RUNNING_TOTAL, AUTO_BUCKET, FUZZY_MATCH, FILL_GAPS are not standard SQL functions.",
|
|
827
|
+
fix: "Use standard SQL: NTILE()/ROW_NUMBER(), STRING_AGG(), FIRST_VALUE() with FILTER, SUM() OVER(), WIDTH_BUCKET(), SIMILARITY() (PostgreSQL).",
|
|
828
|
+
languages: ["sql"],
|
|
829
|
+
},
|
|
487
830
|
];
|
|
488
831
|
// ─── Suspicious Import Patterns ─────────────────────────────────────────────
|
|
489
832
|
/**
|
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
import type { JudgeDefinition, JudgeEvaluation, TribunalVerdict, ProjectVerdict, DiffVerdict,
|
|
1
|
+
import type { JudgeDefinition, JudgeEvaluation, TribunalVerdict, ProjectVerdict, DiffVerdict, MustFixGateOptions, JudgesConfig, StreamingBatch } from "../types.js";
|
|
2
2
|
import type { CodeStructure } from "../ast/types.js";
|
|
3
3
|
import type { TaintFlow } from "../ast/taint-tracker.js";
|
|
4
4
|
import { formatVerdictAsMarkdown, formatEvaluationAsMarkdown } from "./shared.js";
|
|
@@ -122,16 +122,7 @@ export declare function preWarmCaches(files: ReadonlyArray<{
|
|
|
122
122
|
content: string;
|
|
123
123
|
language: string;
|
|
124
124
|
}>, optionsSuffix?: string): void;
|
|
125
|
-
|
|
126
|
-
* Apply inline suppression comments and return both filtered findings
|
|
127
|
-
* and a full audit trail of what was suppressed.
|
|
128
|
-
*/
|
|
129
|
-
export declare function applyInlineSuppressionsWithAudit(findings: Finding[], code: string): SuppressionResult;
|
|
130
|
-
/**
|
|
131
|
-
* Filter findings based on inline suppression comments in the source code.
|
|
132
|
-
* Drop-in backward-compatible wrapper around `applyInlineSuppressionsWithAudit`.
|
|
133
|
-
*/
|
|
134
|
-
export declare function applyInlineSuppressions(findings: Finding[], code: string): Finding[];
|
|
125
|
+
export { applyInlineSuppressions, applyInlineSuppressionsWithAudit } from "./suppressions.js";
|
|
135
126
|
/**
|
|
136
127
|
* Run a single judge against the provided code.
|
|
137
128
|
*/
|
package/dist/evaluators/index.js
CHANGED
|
@@ -201,187 +201,9 @@ function findMatchingTaintFlows(finding, flowsBySinkLine) {
|
|
|
201
201
|
}
|
|
202
202
|
return matched;
|
|
203
203
|
}
|
|
204
|
-
|
|
205
|
-
|
|
206
|
-
|
|
207
|
-
* Supports five directive styles:
|
|
208
|
-
* // judges-ignore RULE-ID → suppress on same line
|
|
209
|
-
* // judges-ignore-next-line RULE-ID → suppress on the next line
|
|
210
|
-
* // judges-ignore-block RULE-ID → suppress until matching end
|
|
211
|
-
* // judges-end-block → ends block suppression
|
|
212
|
-
* // judges-file-ignore RULE-ID → suppress across entire file
|
|
213
|
-
*
|
|
214
|
-
* All directive styles also accept # and /* comment prefixes for
|
|
215
|
-
* Python/YAML/CSS compatibility.
|
|
216
|
-
*
|
|
217
|
-
* An optional reason can be appended after " -- ":
|
|
218
|
-
* // judges-ignore SEC-001 -- legacy code, tracked in JIRA-123
|
|
219
|
-
*/
|
|
220
|
-
function parseInlineSuppressions(code) {
|
|
221
|
-
const lines = code.split("\n");
|
|
222
|
-
const lineSuppressed = new Map();
|
|
223
|
-
const globalSuppressed = [];
|
|
224
|
-
// Active block suppressions: ruleId → { commentLine, reason }
|
|
225
|
-
const activeBlocks = new Map();
|
|
226
|
-
// Pattern: // judges-ignore[-next-line|-block] RULE-ID [, RULE-ID ...] [-- reason]
|
|
227
|
-
const endBlockPattern = /(?:\/\/|#|\/\*)\s*judges-end-block/i;
|
|
228
|
-
for (let i = 0; i < lines.length; i++) {
|
|
229
|
-
const line = lines[i];
|
|
230
|
-
const lineNum = i + 1; // 1-indexed
|
|
231
|
-
// Check for block-end
|
|
232
|
-
if (endBlockPattern.test(line)) {
|
|
233
|
-
activeBlocks.clear();
|
|
234
|
-
}
|
|
235
|
-
// Apply any active block suppressions to this line
|
|
236
|
-
for (const [ruleId, meta] of activeBlocks) {
|
|
237
|
-
const arr = lineSuppressed.get(lineNum) ?? [];
|
|
238
|
-
arr.push({ ruleId, kind: "block", commentLine: meta.commentLine, reason: meta.reason });
|
|
239
|
-
lineSuppressed.set(lineNum, arr);
|
|
240
|
-
}
|
|
241
|
-
// Parse suppression directives (string-based to avoid regex redos)
|
|
242
|
-
const ignoreIdx = line.indexOf("judges-ignore");
|
|
243
|
-
if (ignoreIdx >= 0) {
|
|
244
|
-
const before = line.substring(0, ignoreIdx).trimEnd();
|
|
245
|
-
if (before.endsWith("//") || before.endsWith("#") || before.endsWith("/*")) {
|
|
246
|
-
let rest = line.substring(ignoreIdx + "judges-ignore".length);
|
|
247
|
-
let modifier;
|
|
248
|
-
if (rest.toLowerCase().startsWith("-next-line")) {
|
|
249
|
-
modifier = "next-line";
|
|
250
|
-
rest = rest.substring("-next-line".length);
|
|
251
|
-
}
|
|
252
|
-
else if (rest.toLowerCase().startsWith("-block")) {
|
|
253
|
-
modifier = "block";
|
|
254
|
-
rest = rest.substring("-block".length);
|
|
255
|
-
}
|
|
256
|
-
const trimmedRest = rest.trimStart();
|
|
257
|
-
if (trimmedRest.length < rest.length && trimmedRest.length > 0) {
|
|
258
|
-
let rawContent = trimmedRest;
|
|
259
|
-
if (rawContent.trimEnd().endsWith("*/")) {
|
|
260
|
-
rawContent = rawContent.replace("*/", "").trimEnd();
|
|
261
|
-
}
|
|
262
|
-
const dashSplit = rawContent.split(" -- ");
|
|
263
|
-
const ruleIds = dashSplit[0].split(/[, \t]+/).filter(Boolean);
|
|
264
|
-
const reason = dashSplit[1]?.trim() || undefined;
|
|
265
|
-
const kind = modifier === "next-line" ? "next-line" : modifier === "block" ? "block" : "line";
|
|
266
|
-
const targetLine = kind === "next-line" ? lineNum + 1 : lineNum;
|
|
267
|
-
for (const rawId of ruleIds) {
|
|
268
|
-
const ruleId = rawId === "*" ? "*" : rawId.toUpperCase();
|
|
269
|
-
if (kind === "block") {
|
|
270
|
-
// Start block suppression — applies to all subsequent lines until end-block
|
|
271
|
-
activeBlocks.set(ruleId, { commentLine: lineNum, reason });
|
|
272
|
-
}
|
|
273
|
-
else {
|
|
274
|
-
const arr = lineSuppressed.get(targetLine) ?? [];
|
|
275
|
-
arr.push({ ruleId, kind, commentLine: lineNum, reason });
|
|
276
|
-
lineSuppressed.set(targetLine, arr);
|
|
277
|
-
}
|
|
278
|
-
}
|
|
279
|
-
}
|
|
280
|
-
}
|
|
281
|
-
}
|
|
282
|
-
// File-level suppression: // judges-file-ignore RULE-ID [-- reason]
|
|
283
|
-
const fileIgnoreIdx = line.indexOf("judges-file-ignore");
|
|
284
|
-
if (fileIgnoreIdx >= 0) {
|
|
285
|
-
const beforeFile = line.substring(0, fileIgnoreIdx).trimEnd();
|
|
286
|
-
if (beforeFile.endsWith("//") || beforeFile.endsWith("#") || beforeFile.endsWith("/*")) {
|
|
287
|
-
const fileRest = line.substring(fileIgnoreIdx + "judges-file-ignore".length);
|
|
288
|
-
const fileTrimmedRest = fileRest.trimStart();
|
|
289
|
-
if (fileTrimmedRest.length < fileRest.length && fileTrimmedRest.length > 0) {
|
|
290
|
-
let rawFileContent = fileTrimmedRest;
|
|
291
|
-
if (rawFileContent.trimEnd().endsWith("*/")) {
|
|
292
|
-
rawFileContent = rawFileContent.replace("*/", "").trimEnd();
|
|
293
|
-
}
|
|
294
|
-
const fileDashSplit = rawFileContent.split(" -- ");
|
|
295
|
-
const ruleIds = fileDashSplit[0].split(/[, \t]+/).filter(Boolean);
|
|
296
|
-
const reason = fileDashSplit[1]?.trim() || undefined;
|
|
297
|
-
for (const rawId of ruleIds) {
|
|
298
|
-
const ruleId = rawId === "*" ? "*" : rawId.toUpperCase();
|
|
299
|
-
globalSuppressed.push({ ruleId, kind: "file", commentLine: lineNum, reason });
|
|
300
|
-
}
|
|
301
|
-
}
|
|
302
|
-
}
|
|
303
|
-
}
|
|
304
|
-
}
|
|
305
|
-
return { lineSuppressed, globalSuppressed };
|
|
306
|
-
}
|
|
307
|
-
/**
|
|
308
|
-
* Check whether a rule ID matches a set of suppression directives.
|
|
309
|
-
* Supports exact match, wildcard "*", and prefix wildcards like "AUTH-*".
|
|
310
|
-
*/
|
|
311
|
-
function matchesSuppression(ruleUpper, directives) {
|
|
312
|
-
for (const d of directives) {
|
|
313
|
-
if (d.ruleId === "*" || d.ruleId === ruleUpper) {
|
|
314
|
-
return d;
|
|
315
|
-
}
|
|
316
|
-
if (d.ruleId.endsWith("-*") && ruleUpper.startsWith(d.ruleId.slice(0, -1))) {
|
|
317
|
-
return d;
|
|
318
|
-
}
|
|
319
|
-
}
|
|
320
|
-
return undefined;
|
|
321
|
-
}
|
|
322
|
-
/**
|
|
323
|
-
* Apply inline suppression comments and return both filtered findings
|
|
324
|
-
* and a full audit trail of what was suppressed.
|
|
325
|
-
*/
|
|
326
|
-
export function applyInlineSuppressionsWithAudit(findings, code) {
|
|
327
|
-
const { lineSuppressed, globalSuppressed } = parseInlineSuppressions(code);
|
|
328
|
-
if (lineSuppressed.size === 0 && globalSuppressed.length === 0) {
|
|
329
|
-
return { findings, suppressed: [] };
|
|
330
|
-
}
|
|
331
|
-
const kept = [];
|
|
332
|
-
const suppressed = [];
|
|
333
|
-
for (const f of findings) {
|
|
334
|
-
const ruleUpper = f.ruleId.toUpperCase();
|
|
335
|
-
// Check file-level suppression
|
|
336
|
-
const globalMatch = matchesSuppression(ruleUpper, globalSuppressed);
|
|
337
|
-
if (globalMatch) {
|
|
338
|
-
suppressed.push({
|
|
339
|
-
ruleId: f.ruleId,
|
|
340
|
-
severity: f.severity,
|
|
341
|
-
title: f.title,
|
|
342
|
-
kind: globalMatch.kind,
|
|
343
|
-
commentLine: globalMatch.commentLine,
|
|
344
|
-
findingLines: f.lineNumbers,
|
|
345
|
-
reason: globalMatch.reason,
|
|
346
|
-
});
|
|
347
|
-
continue;
|
|
348
|
-
}
|
|
349
|
-
// Check line-level suppressions
|
|
350
|
-
let wasLineSuppressed = false;
|
|
351
|
-
if (f.lineNumbers && f.lineNumbers.length > 0) {
|
|
352
|
-
for (const lineNum of f.lineNumbers) {
|
|
353
|
-
const directives = lineSuppressed.get(lineNum);
|
|
354
|
-
if (directives) {
|
|
355
|
-
const lineMatch = matchesSuppression(ruleUpper, directives);
|
|
356
|
-
if (lineMatch) {
|
|
357
|
-
suppressed.push({
|
|
358
|
-
ruleId: f.ruleId,
|
|
359
|
-
severity: f.severity,
|
|
360
|
-
title: f.title,
|
|
361
|
-
kind: lineMatch.kind,
|
|
362
|
-
commentLine: lineMatch.commentLine,
|
|
363
|
-
findingLines: f.lineNumbers,
|
|
364
|
-
reason: lineMatch.reason,
|
|
365
|
-
});
|
|
366
|
-
wasLineSuppressed = true;
|
|
367
|
-
break;
|
|
368
|
-
}
|
|
369
|
-
}
|
|
370
|
-
}
|
|
371
|
-
}
|
|
372
|
-
if (!wasLineSuppressed) {
|
|
373
|
-
kept.push(f);
|
|
374
|
-
}
|
|
375
|
-
}
|
|
376
|
-
return { findings: kept, suppressed };
|
|
377
|
-
}
|
|
378
|
-
/**
|
|
379
|
-
* Filter findings based on inline suppression comments in the source code.
|
|
380
|
-
* Drop-in backward-compatible wrapper around `applyInlineSuppressionsWithAudit`.
|
|
381
|
-
*/
|
|
382
|
-
export function applyInlineSuppressions(findings, code) {
|
|
383
|
-
return applyInlineSuppressionsWithAudit(findings, code).findings;
|
|
384
|
-
}
|
|
204
|
+
// ── Inline suppression support (delegated to ./suppressions.ts) ─────────────
|
|
205
|
+
export { applyInlineSuppressions, applyInlineSuppressionsWithAudit } from "./suppressions.js";
|
|
206
|
+
import { applyInlineSuppressionsWithAudit } from "./suppressions.js";
|
|
385
207
|
function resolveJudgeSet(options) {
|
|
386
208
|
const includeAstFindings = options?.includeAstFindings ?? true;
|
|
387
209
|
let judges = includeAstFindings ? JUDGES : JUDGES.filter((judge) => judge.id !== "code-structure");
|