@walkeros/server-transformer-bot 4.4.0 → 4.5.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +25 -0
- package/README.md +283 -53
- package/dist/dev.d.mts +108 -9
- package/dist/dev.d.ts +108 -9
- package/dist/dev.js +1 -1
- package/dist/dev.js.map +1 -1
- package/dist/dev.mjs +1 -1
- package/dist/dev.mjs.map +1 -1
- package/dist/index.d.mts +147 -7
- package/dist/index.d.ts +147 -7
- package/dist/index.js +1 -1
- package/dist/index.js.map +1 -1
- package/dist/index.mjs +1 -1
- package/dist/index.mjs.map +1 -1
- package/dist/walkerOS.json +759 -38
- package/package.json +2 -2
package/dist/walkerOS.json
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
{
|
|
2
2
|
"$meta": {
|
|
3
3
|
"package": "@walkeros/server-transformer-bot",
|
|
4
|
-
"version": "4.
|
|
4
|
+
"version": "4.5.0",
|
|
5
5
|
"type": "transformer",
|
|
6
6
|
"platform": [
|
|
7
7
|
"server"
|
|
@@ -15,7 +15,7 @@
|
|
|
15
15
|
"type": "object",
|
|
16
16
|
"properties": {
|
|
17
17
|
"input": {
|
|
18
|
-
"description": "Input signal sources, resolved via getMappingValue against { event, ingest }.
|
|
18
|
+
"description": "Input signal sources, resolved via getMappingValue against { event, ingest }. Each defaults to \"ingest.<name>\". Listing a name here also declares that the signal is wired in your pipeline, which is what enables the absence-based checks for its family (client hints, Fetch Metadata, Accept-Language/Encoding). \"ja4\" and \"headerNames\" are reserved and unconsumed.",
|
|
19
19
|
"type": "object",
|
|
20
20
|
"properties": {
|
|
21
21
|
"userAgent": {
|
|
@@ -589,41 +589,502 @@
|
|
|
589
589
|
"description": "Array of fallback values, tried in order"
|
|
590
590
|
}
|
|
591
591
|
]
|
|
592
|
+
},
|
|
593
|
+
"accept": {
|
|
594
|
+
"anyOf": [
|
|
595
|
+
{
|
|
596
|
+
"anyOf": [
|
|
597
|
+
{
|
|
598
|
+
"type": "string",
|
|
599
|
+
"description": "Dot-notation path like \"ingest.userAgent\""
|
|
600
|
+
},
|
|
601
|
+
{
|
|
602
|
+
"type": "object",
|
|
603
|
+
"properties": {
|
|
604
|
+
"key": {
|
|
605
|
+
"type": "string"
|
|
606
|
+
},
|
|
607
|
+
"value": {},
|
|
608
|
+
"fn": {
|
|
609
|
+
"type": "string"
|
|
610
|
+
}
|
|
611
|
+
},
|
|
612
|
+
"additionalProperties": false,
|
|
613
|
+
"description": "Mapping value object"
|
|
614
|
+
}
|
|
615
|
+
]
|
|
616
|
+
},
|
|
617
|
+
{
|
|
618
|
+
"type": "array",
|
|
619
|
+
"items": {
|
|
620
|
+
"anyOf": [
|
|
621
|
+
{
|
|
622
|
+
"type": "string",
|
|
623
|
+
"description": "Dot-notation path like \"ingest.userAgent\""
|
|
624
|
+
},
|
|
625
|
+
{
|
|
626
|
+
"type": "object",
|
|
627
|
+
"properties": {
|
|
628
|
+
"key": {
|
|
629
|
+
"type": "string"
|
|
630
|
+
},
|
|
631
|
+
"value": {},
|
|
632
|
+
"fn": {
|
|
633
|
+
"type": "string"
|
|
634
|
+
}
|
|
635
|
+
},
|
|
636
|
+
"additionalProperties": false,
|
|
637
|
+
"description": "Mapping value object"
|
|
638
|
+
}
|
|
639
|
+
]
|
|
640
|
+
},
|
|
641
|
+
"description": "Array of fallback values, tried in order"
|
|
642
|
+
}
|
|
643
|
+
]
|
|
644
|
+
},
|
|
645
|
+
"contentType": {
|
|
646
|
+
"anyOf": [
|
|
647
|
+
{
|
|
648
|
+
"anyOf": [
|
|
649
|
+
{
|
|
650
|
+
"type": "string",
|
|
651
|
+
"description": "Dot-notation path like \"ingest.userAgent\""
|
|
652
|
+
},
|
|
653
|
+
{
|
|
654
|
+
"type": "object",
|
|
655
|
+
"properties": {
|
|
656
|
+
"key": {
|
|
657
|
+
"type": "string"
|
|
658
|
+
},
|
|
659
|
+
"value": {},
|
|
660
|
+
"fn": {
|
|
661
|
+
"type": "string"
|
|
662
|
+
}
|
|
663
|
+
},
|
|
664
|
+
"additionalProperties": false,
|
|
665
|
+
"description": "Mapping value object"
|
|
666
|
+
}
|
|
667
|
+
]
|
|
668
|
+
},
|
|
669
|
+
{
|
|
670
|
+
"type": "array",
|
|
671
|
+
"items": {
|
|
672
|
+
"anyOf": [
|
|
673
|
+
{
|
|
674
|
+
"type": "string",
|
|
675
|
+
"description": "Dot-notation path like \"ingest.userAgent\""
|
|
676
|
+
},
|
|
677
|
+
{
|
|
678
|
+
"type": "object",
|
|
679
|
+
"properties": {
|
|
680
|
+
"key": {
|
|
681
|
+
"type": "string"
|
|
682
|
+
},
|
|
683
|
+
"value": {},
|
|
684
|
+
"fn": {
|
|
685
|
+
"type": "string"
|
|
686
|
+
}
|
|
687
|
+
},
|
|
688
|
+
"additionalProperties": false,
|
|
689
|
+
"description": "Mapping value object"
|
|
690
|
+
}
|
|
691
|
+
]
|
|
692
|
+
},
|
|
693
|
+
"description": "Array of fallback values, tried in order"
|
|
694
|
+
}
|
|
695
|
+
]
|
|
696
|
+
},
|
|
697
|
+
"referer": {
|
|
698
|
+
"anyOf": [
|
|
699
|
+
{
|
|
700
|
+
"anyOf": [
|
|
701
|
+
{
|
|
702
|
+
"type": "string",
|
|
703
|
+
"description": "Dot-notation path like \"ingest.userAgent\""
|
|
704
|
+
},
|
|
705
|
+
{
|
|
706
|
+
"type": "object",
|
|
707
|
+
"properties": {
|
|
708
|
+
"key": {
|
|
709
|
+
"type": "string"
|
|
710
|
+
},
|
|
711
|
+
"value": {},
|
|
712
|
+
"fn": {
|
|
713
|
+
"type": "string"
|
|
714
|
+
}
|
|
715
|
+
},
|
|
716
|
+
"additionalProperties": false,
|
|
717
|
+
"description": "Mapping value object"
|
|
718
|
+
}
|
|
719
|
+
]
|
|
720
|
+
},
|
|
721
|
+
{
|
|
722
|
+
"type": "array",
|
|
723
|
+
"items": {
|
|
724
|
+
"anyOf": [
|
|
725
|
+
{
|
|
726
|
+
"type": "string",
|
|
727
|
+
"description": "Dot-notation path like \"ingest.userAgent\""
|
|
728
|
+
},
|
|
729
|
+
{
|
|
730
|
+
"type": "object",
|
|
731
|
+
"properties": {
|
|
732
|
+
"key": {
|
|
733
|
+
"type": "string"
|
|
734
|
+
},
|
|
735
|
+
"value": {},
|
|
736
|
+
"fn": {
|
|
737
|
+
"type": "string"
|
|
738
|
+
}
|
|
739
|
+
},
|
|
740
|
+
"additionalProperties": false,
|
|
741
|
+
"description": "Mapping value object"
|
|
742
|
+
}
|
|
743
|
+
]
|
|
744
|
+
},
|
|
745
|
+
"description": "Array of fallback values, tried in order"
|
|
746
|
+
}
|
|
747
|
+
]
|
|
748
|
+
},
|
|
749
|
+
"signatureAgent": {
|
|
750
|
+
"anyOf": [
|
|
751
|
+
{
|
|
752
|
+
"anyOf": [
|
|
753
|
+
{
|
|
754
|
+
"type": "string",
|
|
755
|
+
"description": "Dot-notation path like \"ingest.userAgent\""
|
|
756
|
+
},
|
|
757
|
+
{
|
|
758
|
+
"type": "object",
|
|
759
|
+
"properties": {
|
|
760
|
+
"key": {
|
|
761
|
+
"type": "string"
|
|
762
|
+
},
|
|
763
|
+
"value": {},
|
|
764
|
+
"fn": {
|
|
765
|
+
"type": "string"
|
|
766
|
+
}
|
|
767
|
+
},
|
|
768
|
+
"additionalProperties": false,
|
|
769
|
+
"description": "Mapping value object"
|
|
770
|
+
}
|
|
771
|
+
]
|
|
772
|
+
},
|
|
773
|
+
{
|
|
774
|
+
"type": "array",
|
|
775
|
+
"items": {
|
|
776
|
+
"anyOf": [
|
|
777
|
+
{
|
|
778
|
+
"type": "string",
|
|
779
|
+
"description": "Dot-notation path like \"ingest.userAgent\""
|
|
780
|
+
},
|
|
781
|
+
{
|
|
782
|
+
"type": "object",
|
|
783
|
+
"properties": {
|
|
784
|
+
"key": {
|
|
785
|
+
"type": "string"
|
|
786
|
+
},
|
|
787
|
+
"value": {},
|
|
788
|
+
"fn": {
|
|
789
|
+
"type": "string"
|
|
790
|
+
}
|
|
791
|
+
},
|
|
792
|
+
"additionalProperties": false,
|
|
793
|
+
"description": "Mapping value object"
|
|
794
|
+
}
|
|
795
|
+
]
|
|
796
|
+
},
|
|
797
|
+
"description": "Array of fallback values, tried in order"
|
|
798
|
+
}
|
|
799
|
+
]
|
|
800
|
+
},
|
|
801
|
+
"method": {
|
|
802
|
+
"anyOf": [
|
|
803
|
+
{
|
|
804
|
+
"anyOf": [
|
|
805
|
+
{
|
|
806
|
+
"type": "string",
|
|
807
|
+
"description": "Dot-notation path like \"ingest.userAgent\""
|
|
808
|
+
},
|
|
809
|
+
{
|
|
810
|
+
"type": "object",
|
|
811
|
+
"properties": {
|
|
812
|
+
"key": {
|
|
813
|
+
"type": "string"
|
|
814
|
+
},
|
|
815
|
+
"value": {},
|
|
816
|
+
"fn": {
|
|
817
|
+
"type": "string"
|
|
818
|
+
}
|
|
819
|
+
},
|
|
820
|
+
"additionalProperties": false,
|
|
821
|
+
"description": "Mapping value object"
|
|
822
|
+
}
|
|
823
|
+
]
|
|
824
|
+
},
|
|
825
|
+
{
|
|
826
|
+
"type": "array",
|
|
827
|
+
"items": {
|
|
828
|
+
"anyOf": [
|
|
829
|
+
{
|
|
830
|
+
"type": "string",
|
|
831
|
+
"description": "Dot-notation path like \"ingest.userAgent\""
|
|
832
|
+
},
|
|
833
|
+
{
|
|
834
|
+
"type": "object",
|
|
835
|
+
"properties": {
|
|
836
|
+
"key": {
|
|
837
|
+
"type": "string"
|
|
838
|
+
},
|
|
839
|
+
"value": {},
|
|
840
|
+
"fn": {
|
|
841
|
+
"type": "string"
|
|
842
|
+
}
|
|
843
|
+
},
|
|
844
|
+
"additionalProperties": false,
|
|
845
|
+
"description": "Mapping value object"
|
|
846
|
+
}
|
|
847
|
+
]
|
|
848
|
+
},
|
|
849
|
+
"description": "Array of fallback values, tried in order"
|
|
850
|
+
}
|
|
851
|
+
]
|
|
852
|
+
},
|
|
853
|
+
"ja4": {
|
|
854
|
+
"anyOf": [
|
|
855
|
+
{
|
|
856
|
+
"anyOf": [
|
|
857
|
+
{
|
|
858
|
+
"type": "string",
|
|
859
|
+
"description": "Dot-notation path like \"ingest.userAgent\""
|
|
860
|
+
},
|
|
861
|
+
{
|
|
862
|
+
"type": "object",
|
|
863
|
+
"properties": {
|
|
864
|
+
"key": {
|
|
865
|
+
"type": "string"
|
|
866
|
+
},
|
|
867
|
+
"value": {},
|
|
868
|
+
"fn": {
|
|
869
|
+
"type": "string"
|
|
870
|
+
}
|
|
871
|
+
},
|
|
872
|
+
"additionalProperties": false,
|
|
873
|
+
"description": "Mapping value object"
|
|
874
|
+
}
|
|
875
|
+
]
|
|
876
|
+
},
|
|
877
|
+
{
|
|
878
|
+
"type": "array",
|
|
879
|
+
"items": {
|
|
880
|
+
"anyOf": [
|
|
881
|
+
{
|
|
882
|
+
"type": "string",
|
|
883
|
+
"description": "Dot-notation path like \"ingest.userAgent\""
|
|
884
|
+
},
|
|
885
|
+
{
|
|
886
|
+
"type": "object",
|
|
887
|
+
"properties": {
|
|
888
|
+
"key": {
|
|
889
|
+
"type": "string"
|
|
890
|
+
},
|
|
891
|
+
"value": {},
|
|
892
|
+
"fn": {
|
|
893
|
+
"type": "string"
|
|
894
|
+
}
|
|
895
|
+
},
|
|
896
|
+
"additionalProperties": false,
|
|
897
|
+
"description": "Mapping value object"
|
|
898
|
+
}
|
|
899
|
+
]
|
|
900
|
+
},
|
|
901
|
+
"description": "Array of fallback values, tried in order"
|
|
902
|
+
}
|
|
903
|
+
]
|
|
904
|
+
},
|
|
905
|
+
"headerNames": {
|
|
906
|
+
"anyOf": [
|
|
907
|
+
{
|
|
908
|
+
"anyOf": [
|
|
909
|
+
{
|
|
910
|
+
"type": "string",
|
|
911
|
+
"description": "Dot-notation path like \"ingest.userAgent\""
|
|
912
|
+
},
|
|
913
|
+
{
|
|
914
|
+
"type": "object",
|
|
915
|
+
"properties": {
|
|
916
|
+
"key": {
|
|
917
|
+
"type": "string"
|
|
918
|
+
},
|
|
919
|
+
"value": {},
|
|
920
|
+
"fn": {
|
|
921
|
+
"type": "string"
|
|
922
|
+
}
|
|
923
|
+
},
|
|
924
|
+
"additionalProperties": false,
|
|
925
|
+
"description": "Mapping value object"
|
|
926
|
+
}
|
|
927
|
+
]
|
|
928
|
+
},
|
|
929
|
+
{
|
|
930
|
+
"type": "array",
|
|
931
|
+
"items": {
|
|
932
|
+
"anyOf": [
|
|
933
|
+
{
|
|
934
|
+
"type": "string",
|
|
935
|
+
"description": "Dot-notation path like \"ingest.userAgent\""
|
|
936
|
+
},
|
|
937
|
+
{
|
|
938
|
+
"type": "object",
|
|
939
|
+
"properties": {
|
|
940
|
+
"key": {
|
|
941
|
+
"type": "string"
|
|
942
|
+
},
|
|
943
|
+
"value": {},
|
|
944
|
+
"fn": {
|
|
945
|
+
"type": "string"
|
|
946
|
+
}
|
|
947
|
+
},
|
|
948
|
+
"additionalProperties": false,
|
|
949
|
+
"description": "Mapping value object"
|
|
950
|
+
}
|
|
951
|
+
]
|
|
952
|
+
},
|
|
953
|
+
"description": "Array of fallback values, tried in order"
|
|
954
|
+
}
|
|
955
|
+
]
|
|
956
|
+
}
|
|
957
|
+
},
|
|
958
|
+
"additionalProperties": false,
|
|
959
|
+
"title": "input"
|
|
960
|
+
},
|
|
961
|
+
"output": {
|
|
962
|
+
"description": "Output paths for the bot annotations.",
|
|
963
|
+
"type": "object",
|
|
964
|
+
"properties": {
|
|
965
|
+
"botScore": {
|
|
966
|
+
"description": "Path for the automation likelihood (0-99, higher = more automated, null when not measured). Default: \"user.botScore\". Use \"ingest.*\" to route to pipeline scratch instead of the event, or false to disable.",
|
|
967
|
+
"anyOf": [
|
|
968
|
+
{
|
|
969
|
+
"type": "string"
|
|
970
|
+
},
|
|
971
|
+
{
|
|
972
|
+
"type": "boolean",
|
|
973
|
+
"const": false
|
|
974
|
+
}
|
|
975
|
+
]
|
|
976
|
+
},
|
|
977
|
+
"botCategory": {
|
|
978
|
+
"description": "Path for the client category: human, suspicious, automation, search-crawler, seo-tool, monitor, link-preview, ai-agent, ai-crawler, unknown. Default: \"user.botCategory\".",
|
|
979
|
+
"anyOf": [
|
|
980
|
+
{
|
|
981
|
+
"type": "string"
|
|
982
|
+
},
|
|
983
|
+
{
|
|
984
|
+
"type": "boolean",
|
|
985
|
+
"const": false
|
|
986
|
+
}
|
|
987
|
+
]
|
|
988
|
+
},
|
|
989
|
+
"botProduct": {
|
|
990
|
+
"description": "Path for the identified product (e.g. \"ChatGPT-User\", \"Googlebot\"), written only when a named detector matched. Default: \"user.botProduct\".",
|
|
991
|
+
"anyOf": [
|
|
992
|
+
{
|
|
993
|
+
"type": "string"
|
|
994
|
+
},
|
|
995
|
+
{
|
|
996
|
+
"type": "boolean",
|
|
997
|
+
"const": false
|
|
998
|
+
}
|
|
999
|
+
]
|
|
1000
|
+
},
|
|
1001
|
+
"botReasons": {
|
|
1002
|
+
"description": "Path for the reason-code array. Default: \"ingest.bot.reasons\", so the codes stay available to the pipeline without weighting the analytics payload. Codes ending in _not_declared report which signal families are unwired.",
|
|
1003
|
+
"anyOf": [
|
|
1004
|
+
{
|
|
1005
|
+
"type": "string"
|
|
1006
|
+
},
|
|
1007
|
+
{
|
|
1008
|
+
"type": "boolean",
|
|
1009
|
+
"const": false
|
|
1010
|
+
}
|
|
1011
|
+
]
|
|
592
1012
|
}
|
|
593
1013
|
},
|
|
594
1014
|
"additionalProperties": false,
|
|
595
|
-
"title": "
|
|
1015
|
+
"title": "output"
|
|
596
1016
|
},
|
|
597
|
-
"
|
|
598
|
-
"description": "
|
|
599
|
-
"
|
|
600
|
-
|
|
601
|
-
|
|
602
|
-
"
|
|
603
|
-
|
|
1017
|
+
"context": {
|
|
1018
|
+
"description": "How the request reaches the collector. An enum literal (\"beacon\") pins one context for every request. Any other form is a Mapping.Value resolved per request against { event, ingest }: a dot-path string (\"ingest.transport\"), a {key}/{value}/{fn} object, or a fallback array tried in order ([{key: \"ingest.transport\"}, {value: \"beacon\"}]). Wire transport truth in via the source config, e.g. express ingest map transport: {key: \"query.transport\"} with a ?transport= param on the collect URL. A result that is not a valid context falls back to \"auto\" (context-independent checks only, reported as \"context_undetermined\"): scored less, never scored wrong.",
|
|
1019
|
+
"anyOf": [
|
|
1020
|
+
{
|
|
1021
|
+
"type": "string",
|
|
1022
|
+
"enum": [
|
|
1023
|
+
"auto",
|
|
1024
|
+
"navigation",
|
|
1025
|
+
"pixel",
|
|
1026
|
+
"beacon",
|
|
1027
|
+
"fetch",
|
|
1028
|
+
"server"
|
|
1029
|
+
]
|
|
1030
|
+
},
|
|
1031
|
+
{
|
|
1032
|
+
"type": "string",
|
|
1033
|
+
"pattern": "\\."
|
|
604
1034
|
},
|
|
605
|
-
|
|
606
|
-
"
|
|
607
|
-
"
|
|
1035
|
+
{
|
|
1036
|
+
"type": "object",
|
|
1037
|
+
"properties": {
|
|
1038
|
+
"key": {
|
|
1039
|
+
"type": "string"
|
|
1040
|
+
},
|
|
1041
|
+
"value": {},
|
|
1042
|
+
"fn": {
|
|
1043
|
+
"type": "string"
|
|
1044
|
+
}
|
|
1045
|
+
},
|
|
1046
|
+
"additionalProperties": false
|
|
608
1047
|
},
|
|
609
|
-
|
|
610
|
-
"
|
|
611
|
-
"
|
|
1048
|
+
{
|
|
1049
|
+
"type": "array",
|
|
1050
|
+
"items": {
|
|
1051
|
+
"anyOf": [
|
|
1052
|
+
{
|
|
1053
|
+
"type": "string",
|
|
1054
|
+
"pattern": "\\."
|
|
1055
|
+
},
|
|
1056
|
+
{
|
|
1057
|
+
"type": "object",
|
|
1058
|
+
"properties": {
|
|
1059
|
+
"key": {
|
|
1060
|
+
"type": "string"
|
|
1061
|
+
},
|
|
1062
|
+
"value": {},
|
|
1063
|
+
"fn": {
|
|
1064
|
+
"type": "string"
|
|
1065
|
+
}
|
|
1066
|
+
},
|
|
1067
|
+
"additionalProperties": false
|
|
1068
|
+
}
|
|
1069
|
+
]
|
|
1070
|
+
}
|
|
612
1071
|
}
|
|
613
|
-
|
|
614
|
-
|
|
615
|
-
|
|
1072
|
+
]
|
|
1073
|
+
},
|
|
1074
|
+
"suspiciousAt": {
|
|
1075
|
+
"description": "Graded-layer cut between category \"human\" and \"suspicious\". Default: 25. Does not affect the deterministic scores (70 and above).",
|
|
1076
|
+
"type": "number"
|
|
616
1077
|
}
|
|
617
1078
|
},
|
|
618
1079
|
"additionalProperties": false,
|
|
619
|
-
"description": "Bot detection transformer: annotates events with
|
|
1080
|
+
"description": "Bot detection transformer: annotates events with an automation likelihood, a client category, the identified product and reason codes."
|
|
620
1081
|
}
|
|
621
1082
|
},
|
|
622
1083
|
"examples": {
|
|
623
1084
|
"step": {
|
|
624
1085
|
"chatgptUserAgent": {
|
|
625
|
-
"title": "ChatGPT-User (
|
|
626
|
-
"description": "A
|
|
1086
|
+
"title": "ChatGPT-User (AI agent)",
|
|
1087
|
+
"description": "A person routed an AI to fetch this page. Same score as a crawler because it is still software; the category is what lets a destination keep this traffic.",
|
|
627
1088
|
"in": {
|
|
628
1089
|
"name": "page view",
|
|
629
1090
|
"data": {
|
|
@@ -661,7 +1122,8 @@
|
|
|
661
1122
|
},
|
|
662
1123
|
"user": {
|
|
663
1124
|
"botScore": 90,
|
|
664
|
-
"
|
|
1125
|
+
"botCategory": "ai-agent",
|
|
1126
|
+
"botProduct": "ChatGPT-User"
|
|
665
1127
|
}
|
|
666
1128
|
}
|
|
667
1129
|
}
|
|
@@ -670,7 +1132,7 @@
|
|
|
670
1132
|
},
|
|
671
1133
|
"curlClient": {
|
|
672
1134
|
"public": false,
|
|
673
|
-
"description": "curl
|
|
1135
|
+
"description": "curl. isbot recognises it as automated without naming a product, so it sits a rung below the UA maps.",
|
|
674
1136
|
"in": {
|
|
675
1137
|
"name": "page view",
|
|
676
1138
|
"data": {
|
|
@@ -708,7 +1170,7 @@
|
|
|
708
1170
|
},
|
|
709
1171
|
"user": {
|
|
710
1172
|
"botScore": 80,
|
|
711
|
-
"
|
|
1173
|
+
"botCategory": "automation"
|
|
712
1174
|
}
|
|
713
1175
|
}
|
|
714
1176
|
}
|
|
@@ -717,7 +1179,7 @@
|
|
|
717
1179
|
},
|
|
718
1180
|
"gptBotCrawler": {
|
|
719
1181
|
"title": "GPTBot training crawler",
|
|
720
|
-
"description": "OpenAI training crawler.
|
|
1182
|
+
"description": "OpenAI training crawler. The category says what it is, the product says which one.",
|
|
721
1183
|
"in": {
|
|
722
1184
|
"name": "page view",
|
|
723
1185
|
"data": {
|
|
@@ -754,8 +1216,56 @@
|
|
|
754
1216
|
"platform": "server"
|
|
755
1217
|
},
|
|
756
1218
|
"user": {
|
|
757
|
-
"botScore":
|
|
758
|
-
"
|
|
1219
|
+
"botScore": 90,
|
|
1220
|
+
"botCategory": "ai-crawler",
|
|
1221
|
+
"botProduct": "GPTBot"
|
|
1222
|
+
}
|
|
1223
|
+
}
|
|
1224
|
+
}
|
|
1225
|
+
]
|
|
1226
|
+
]
|
|
1227
|
+
},
|
|
1228
|
+
"headerMismatch": {
|
|
1229
|
+
"title": "Client hints contradict the UA",
|
|
1230
|
+
"description": "The UA claims Chrome 124, Sec-CH-UA says Chromium 98. Graded evidence, not proof: frozen WebView UAs and enterprise UA-reduction policies produce the same mismatch, so the outcome is \"suspicious\" and never \"automation\".",
|
|
1231
|
+
"in": {
|
|
1232
|
+
"name": "page view",
|
|
1233
|
+
"data": {
|
|
1234
|
+
"title": "Home",
|
|
1235
|
+
"id": "/"
|
|
1236
|
+
},
|
|
1237
|
+
"id": "ev-1700000606",
|
|
1238
|
+
"trigger": "load",
|
|
1239
|
+
"entity": "page",
|
|
1240
|
+
"action": "view",
|
|
1241
|
+
"timestamp": 1700000600,
|
|
1242
|
+
"source": {
|
|
1243
|
+
"type": "express",
|
|
1244
|
+
"platform": "server"
|
|
1245
|
+
}
|
|
1246
|
+
},
|
|
1247
|
+
"out": [
|
|
1248
|
+
[
|
|
1249
|
+
"return",
|
|
1250
|
+
{
|
|
1251
|
+
"event": {
|
|
1252
|
+
"name": "page view",
|
|
1253
|
+
"data": {
|
|
1254
|
+
"title": "Home",
|
|
1255
|
+
"id": "/"
|
|
1256
|
+
},
|
|
1257
|
+
"id": "ev-1700000606",
|
|
1258
|
+
"trigger": "load",
|
|
1259
|
+
"entity": "page",
|
|
1260
|
+
"action": "view",
|
|
1261
|
+
"timestamp": 1700000600,
|
|
1262
|
+
"source": {
|
|
1263
|
+
"type": "express",
|
|
1264
|
+
"platform": "server"
|
|
1265
|
+
},
|
|
1266
|
+
"user": {
|
|
1267
|
+
"botScore": 30,
|
|
1268
|
+
"botCategory": "suspicious"
|
|
759
1269
|
}
|
|
760
1270
|
}
|
|
761
1271
|
}
|
|
@@ -764,7 +1274,7 @@
|
|
|
764
1274
|
},
|
|
765
1275
|
"humanChrome": {
|
|
766
1276
|
"title": "Human visitor (Chrome)",
|
|
767
|
-
"description": "Modern Chrome UA. No bot
|
|
1277
|
+
"description": "Modern Chrome UA. No bot signals.",
|
|
768
1278
|
"in": {
|
|
769
1279
|
"name": "page view",
|
|
770
1280
|
"data": {
|
|
@@ -802,7 +1312,7 @@
|
|
|
802
1312
|
},
|
|
803
1313
|
"user": {
|
|
804
1314
|
"botScore": 0,
|
|
805
|
-
"
|
|
1315
|
+
"botCategory": "human"
|
|
806
1316
|
}
|
|
807
1317
|
}
|
|
808
1318
|
}
|
|
@@ -811,7 +1321,7 @@
|
|
|
811
1321
|
},
|
|
812
1322
|
"missingUA": {
|
|
813
1323
|
"public": false,
|
|
814
|
-
"description": "No User-Agent
|
|
1324
|
+
"description": "No User-Agent, but the pipeline is delivering other signals. UA stripping is overwhelmingly bots or hardened privacy tools.",
|
|
815
1325
|
"in": {
|
|
816
1326
|
"name": "page view",
|
|
817
1327
|
"data": {
|
|
@@ -849,7 +1359,196 @@
|
|
|
849
1359
|
},
|
|
850
1360
|
"user": {
|
|
851
1361
|
"botScore": 70,
|
|
852
|
-
"
|
|
1362
|
+
"botCategory": "automation"
|
|
1363
|
+
}
|
|
1364
|
+
}
|
|
1365
|
+
}
|
|
1366
|
+
]
|
|
1367
|
+
]
|
|
1368
|
+
},
|
|
1369
|
+
"pixelWildcardAccept": {
|
|
1370
|
+
"title": "Wildcard Accept on an annotated pixel",
|
|
1371
|
+
"description": "The pixel embed URL carries ?transport=pixel and the source lifts it into ingest, so this request is scored against the pixel profile. Browsers send a typed image Accept when they load an image, so the wildcard is worth 25.",
|
|
1372
|
+
"in": {
|
|
1373
|
+
"name": "page view",
|
|
1374
|
+
"data": {
|
|
1375
|
+
"title": "Home",
|
|
1376
|
+
"id": "/"
|
|
1377
|
+
},
|
|
1378
|
+
"id": "ev-1700000608",
|
|
1379
|
+
"trigger": "load",
|
|
1380
|
+
"entity": "page",
|
|
1381
|
+
"action": "view",
|
|
1382
|
+
"timestamp": 1700000600,
|
|
1383
|
+
"source": {
|
|
1384
|
+
"type": "express",
|
|
1385
|
+
"platform": "server"
|
|
1386
|
+
}
|
|
1387
|
+
},
|
|
1388
|
+
"out": [
|
|
1389
|
+
[
|
|
1390
|
+
"return",
|
|
1391
|
+
{
|
|
1392
|
+
"event": {
|
|
1393
|
+
"name": "page view",
|
|
1394
|
+
"data": {
|
|
1395
|
+
"title": "Home",
|
|
1396
|
+
"id": "/"
|
|
1397
|
+
},
|
|
1398
|
+
"id": "ev-1700000608",
|
|
1399
|
+
"trigger": "load",
|
|
1400
|
+
"entity": "page",
|
|
1401
|
+
"action": "view",
|
|
1402
|
+
"timestamp": 1700000600,
|
|
1403
|
+
"source": {
|
|
1404
|
+
"type": "express",
|
|
1405
|
+
"platform": "server"
|
|
1406
|
+
},
|
|
1407
|
+
"user": {
|
|
1408
|
+
"botScore": 25,
|
|
1409
|
+
"botCategory": "suspicious"
|
|
1410
|
+
}
|
|
1411
|
+
}
|
|
1412
|
+
}
|
|
1413
|
+
]
|
|
1414
|
+
]
|
|
1415
|
+
},
|
|
1416
|
+
"searchCrawler": {
|
|
1417
|
+
"title": "Googlebot (search crawler)",
|
|
1418
|
+
"description": "Correlates with organic discoverability, so it is worth separating from both AI crawlers and unnamed automation.",
|
|
1419
|
+
"in": {
|
|
1420
|
+
"name": "page view",
|
|
1421
|
+
"data": {
|
|
1422
|
+
"title": "Home",
|
|
1423
|
+
"id": "/"
|
|
1424
|
+
},
|
|
1425
|
+
"id": "ev-1700000605",
|
|
1426
|
+
"trigger": "load",
|
|
1427
|
+
"entity": "page",
|
|
1428
|
+
"action": "view",
|
|
1429
|
+
"timestamp": 1700000600,
|
|
1430
|
+
"source": {
|
|
1431
|
+
"type": "express",
|
|
1432
|
+
"platform": "server"
|
|
1433
|
+
}
|
|
1434
|
+
},
|
|
1435
|
+
"out": [
|
|
1436
|
+
[
|
|
1437
|
+
"return",
|
|
1438
|
+
{
|
|
1439
|
+
"event": {
|
|
1440
|
+
"name": "page view",
|
|
1441
|
+
"data": {
|
|
1442
|
+
"title": "Home",
|
|
1443
|
+
"id": "/"
|
|
1444
|
+
},
|
|
1445
|
+
"id": "ev-1700000605",
|
|
1446
|
+
"trigger": "load",
|
|
1447
|
+
"entity": "page",
|
|
1448
|
+
"action": "view",
|
|
1449
|
+
"timestamp": 1700000600,
|
|
1450
|
+
"source": {
|
|
1451
|
+
"type": "express",
|
|
1452
|
+
"platform": "server"
|
|
1453
|
+
},
|
|
1454
|
+
"user": {
|
|
1455
|
+
"botScore": 90,
|
|
1456
|
+
"botCategory": "search-crawler",
|
|
1457
|
+
"botProduct": "Googlebot"
|
|
1458
|
+
}
|
|
1459
|
+
}
|
|
1460
|
+
}
|
|
1461
|
+
]
|
|
1462
|
+
]
|
|
1463
|
+
},
|
|
1464
|
+
"unannotatedBeaconFallback": {
|
|
1465
|
+
"title": "Unannotated request falls back to beacon",
|
|
1466
|
+
"description": "Same instance, same headers, no ?transport= on the request, so the trailing { value: \"beacon\" } pins beacon. A wildcard Accept is exactly what navigator.sendBeacon sends, and the identical request now scores 0. This is what pinning per request buys: one header, two correct readings.",
|
|
1467
|
+
"in": {
|
|
1468
|
+
"name": "page view",
|
|
1469
|
+
"data": {
|
|
1470
|
+
"title": "Home",
|
|
1471
|
+
"id": "/"
|
|
1472
|
+
},
|
|
1473
|
+
"id": "ev-1700000609",
|
|
1474
|
+
"trigger": "load",
|
|
1475
|
+
"entity": "page",
|
|
1476
|
+
"action": "view",
|
|
1477
|
+
"timestamp": 1700000600,
|
|
1478
|
+
"source": {
|
|
1479
|
+
"type": "express",
|
|
1480
|
+
"platform": "server"
|
|
1481
|
+
}
|
|
1482
|
+
},
|
|
1483
|
+
"out": [
|
|
1484
|
+
[
|
|
1485
|
+
"return",
|
|
1486
|
+
{
|
|
1487
|
+
"event": {
|
|
1488
|
+
"name": "page view",
|
|
1489
|
+
"data": {
|
|
1490
|
+
"title": "Home",
|
|
1491
|
+
"id": "/"
|
|
1492
|
+
},
|
|
1493
|
+
"id": "ev-1700000609",
|
|
1494
|
+
"trigger": "load",
|
|
1495
|
+
"entity": "page",
|
|
1496
|
+
"action": "view",
|
|
1497
|
+
"timestamp": 1700000600,
|
|
1498
|
+
"source": {
|
|
1499
|
+
"type": "express",
|
|
1500
|
+
"platform": "server"
|
|
1501
|
+
},
|
|
1502
|
+
"user": {
|
|
1503
|
+
"botScore": 0,
|
|
1504
|
+
"botCategory": "human"
|
|
1505
|
+
}
|
|
1506
|
+
}
|
|
1507
|
+
}
|
|
1508
|
+
]
|
|
1509
|
+
]
|
|
1510
|
+
},
|
|
1511
|
+
"unwiredPipeline": {
|
|
1512
|
+
"public": false,
|
|
1513
|
+
"description": "No signal resolved, so the score is null rather than 0. Writing null overwrites any client-supplied value and still evaluates false under a \"botScore > 50\" filter. ingest.bot.reasons names the missing mappings.",
|
|
1514
|
+
"in": {
|
|
1515
|
+
"name": "page view",
|
|
1516
|
+
"data": {
|
|
1517
|
+
"title": "Home",
|
|
1518
|
+
"id": "/"
|
|
1519
|
+
},
|
|
1520
|
+
"id": "ev-1700000607",
|
|
1521
|
+
"trigger": "load",
|
|
1522
|
+
"entity": "page",
|
|
1523
|
+
"action": "view",
|
|
1524
|
+
"timestamp": 1700000600,
|
|
1525
|
+
"source": {
|
|
1526
|
+
"type": "express",
|
|
1527
|
+
"platform": "server"
|
|
1528
|
+
}
|
|
1529
|
+
},
|
|
1530
|
+
"out": [
|
|
1531
|
+
[
|
|
1532
|
+
"return",
|
|
1533
|
+
{
|
|
1534
|
+
"event": {
|
|
1535
|
+
"name": "page view",
|
|
1536
|
+
"data": {
|
|
1537
|
+
"title": "Home",
|
|
1538
|
+
"id": "/"
|
|
1539
|
+
},
|
|
1540
|
+
"id": "ev-1700000607",
|
|
1541
|
+
"trigger": "load",
|
|
1542
|
+
"entity": "page",
|
|
1543
|
+
"action": "view",
|
|
1544
|
+
"timestamp": 1700000600,
|
|
1545
|
+
"source": {
|
|
1546
|
+
"type": "express",
|
|
1547
|
+
"platform": "server"
|
|
1548
|
+
},
|
|
1549
|
+
"user": {
|
|
1550
|
+
"botScore": null,
|
|
1551
|
+
"botCategory": "unknown"
|
|
853
1552
|
}
|
|
854
1553
|
}
|
|
855
1554
|
}
|
|
@@ -860,28 +1559,50 @@
|
|
|
860
1559
|
},
|
|
861
1560
|
"hints": {
|
|
862
1561
|
"ingest-prerequisite": {
|
|
863
|
-
"text": "The bot transformer reads
|
|
1562
|
+
"text": "The bot transformer reads its signals from ctx.ingest (path \"ingest.<name>\" by default). The upstream server source must populate them via config.ingest, which MUST use the map operator with direct request field paths (no req. prefix); a bare object like { userAgent: \"req.headers.user-agent\" } is silently inert and leaves ingest empty. With nothing in ingest the score is null and the category is \"unknown\". The map is also where the request context enters: lift the sender annotation with transport: { key: \"query.transport\" } and one instance then serves every transport, see the transport-wiring hint.",
|
|
1563
|
+
"code": [
|
|
1564
|
+
{
|
|
1565
|
+
"lang": "json",
|
|
1566
|
+
"code": "{\n \"sources\": {\n \"express\": {\n \"package\": \"@walkeros/server-source-express\",\n \"config\": {\n \"ingest\": {\n \"map\": {\n \"userAgent\": {\n \"key\": \"headers.user-agent\"\n },\n \"acceptLanguage\": {\n \"key\": \"headers.accept-language\"\n },\n \"acceptEncoding\": {\n \"key\": \"headers.accept-encoding\"\n },\n \"secFetchSite\": {\n \"key\": \"headers.sec-fetch-site\"\n },\n \"secFetchMode\": {\n \"key\": \"headers.sec-fetch-mode\"\n },\n \"secFetchDest\": {\n \"key\": \"headers.sec-fetch-dest\"\n },\n \"secFetchUser\": {\n \"key\": \"headers.sec-fetch-user\"\n },\n \"secChUa\": {\n \"key\": \"headers.sec-ch-ua\"\n },\n \"secChUaMobile\": {\n \"key\": \"headers.sec-ch-ua-mobile\"\n },\n \"secChUaPlatform\": {\n \"key\": \"headers.sec-ch-ua-platform\"\n },\n \"accept\": {\n \"key\": \"headers.accept\"\n },\n \"contentType\": {\n \"key\": \"headers.content-type\"\n },\n \"referer\": {\n \"key\": \"headers.referer\"\n },\n \"signatureAgent\": {\n \"key\": \"headers.signature-agent\"\n },\n \"method\": {\n \"key\": \"method\"\n },\n \"transport\": {\n \"key\": \"query.transport\"\n }\n }\n }\n }\n }\n },\n \"transformers\": {\n \"bot\": {\n \"package\": \"@walkeros/server-transformer-bot\",\n \"config\": {\n \"settings\": {\n \"context\": [\n {\n \"key\": \"ingest.transport\"\n },\n {\n \"value\": \"beacon\"\n }\n ]\n }\n }\n }\n }\n}"
|
|
1567
|
+
}
|
|
1568
|
+
]
|
|
1569
|
+
},
|
|
1570
|
+
"transport-wiring": {
|
|
1571
|
+
"text": "settings.context accepts a Mapping.Value, so one instance covers a deployment that receives beacons, pixels and fetches at the same endpoint. The sender declares the transport, the source lifts it, the transformer resolves it per request: add ?transport=beacon to the collect URL and ?transport=pixel to the pixel embed URL, then map transport: { key: \"query.transport\" } in the source (the normalized express scope exposes the parsed query, so no extra plumbing is needed). Give the bot config a fallback array so unannotated senders still land on a sane profile: [{ \"key\": \"ingest.transport\" }, { \"value\": \"beacon\" }]. Mind the shape: a bare string is read as a lookup path and must contain a dot, so a literal inside the array is written { \"value\": \"beacon\" } and never \"beacon\". Anything that resolves outside the vocabulary falls back to \"auto\", which scores less rather than scoring wrong. A query parameter is a claim by whoever controls the sender, which is fine for separating your own transports and worthless against a client that wants to be scored as a beacon; when you need server truth instead, give pixels their own route and derive the context from the request path.",
|
|
1572
|
+
"code": [
|
|
1573
|
+
{
|
|
1574
|
+
"lang": "html",
|
|
1575
|
+
"code": "<img src=\"https://collect.example.com/px.gif?transport=pixel\" width=\"1\" height=\"1\" alt=\"\">"
|
|
1576
|
+
},
|
|
1577
|
+
{
|
|
1578
|
+
"lang": "json",
|
|
1579
|
+
"code": "{\n \"sources\": {\n \"express\": {\n \"package\": \"@walkeros/server-source-express\",\n \"config\": {\n \"ingest\": {\n \"map\": {\n \"userAgent\": {\n \"key\": \"headers.user-agent\"\n },\n \"transport\": {\n \"key\": \"query.transport\"\n }\n }\n }\n }\n }\n },\n \"transformers\": {\n \"bot\": {\n \"package\": \"@walkeros/server-transformer-bot\",\n \"config\": {\n \"settings\": {\n \"context\": [\n {\n \"key\": \"ingest.transport\"\n },\n {\n \"value\": \"beacon\"\n }\n ]\n }\n }\n }\n }\n}"
|
|
1580
|
+
}
|
|
1581
|
+
]
|
|
1582
|
+
},
|
|
1583
|
+
"declared-signals": {
|
|
1584
|
+
"text": "Absence is ambiguous: \"the client sent no Sec-CH-UA\" and \"the operator never mapped Sec-CH-UA\" both arrive as undefined. So an absence-based check runs only when its input name appears explicitly in settings.input, which is you asserting the signal is wired. Reading still falls back to the defaults, so listing a name with its default path costs nothing and unlocks the check. Until then ingest.bot.reasons reports ch_not_declared, fetchmeta_not_declared or accept_not_declared, naming exactly which mapping to add.",
|
|
864
1585
|
"code": [
|
|
865
1586
|
{
|
|
866
1587
|
"lang": "json",
|
|
867
|
-
"code": "{\n \"
|
|
1588
|
+
"code": "{\n \"transformers\": {\n \"bot\": {\n \"package\": \"@walkeros/server-transformer-bot\",\n \"config\": {\n \"settings\": {\n \"context\": \"beacon\",\n \"input\": {\n \"acceptLanguage\": \"ingest.acceptLanguage\",\n \"acceptEncoding\": \"ingest.acceptEncoding\",\n \"secFetchSite\": \"ingest.secFetchSite\",\n \"secFetchMode\": \"ingest.secFetchMode\",\n \"secFetchDest\": \"ingest.secFetchDest\",\n \"secChUa\": \"ingest.secChUa\"\n }\n }\n }\n }\n }\n}"
|
|
868
1589
|
}
|
|
869
1590
|
]
|
|
870
1591
|
},
|
|
871
1592
|
"output-routing": {
|
|
872
|
-
"text": "Outputs default to event.user.botScore
|
|
1593
|
+
"text": "Outputs default to event.user.botScore, event.user.botCategory, event.user.botProduct and ingest.bot.reasons. Any value can be a dot path to rename and reroute the field, or false to disable it. An \"ingest.\" prefix keeps a field off the analytics event while still routing on it downstream. Note that ingest is shared by every event in a scope and the write is an assignment, so if a before-chain step fans one input into several events, point botReasons at an event path to keep per-event attribution.",
|
|
873
1594
|
"code": [
|
|
874
1595
|
{
|
|
875
1596
|
"lang": "json",
|
|
876
|
-
"code": "{\n \"transformers\": {\n \"bot\": {\n \"package\": \"@walkeros/server-transformer-bot\",\n \"config\": {\n \"settings\": {\n \"output\": {\n \"botScore\": \"ingest.bot.score\",\n \"
|
|
1597
|
+
"code": "{\n \"transformers\": {\n \"bot\": {\n \"package\": \"@walkeros/server-transformer-bot\",\n \"config\": {\n \"settings\": {\n \"output\": {\n \"botScore\": \"ingest.bot.score\",\n \"botCategory\": \"ingest.bot.category\",\n \"botProduct\": false,\n \"botReasons\": \"ingest.bot.reasons\"\n }\n }\n }\n }\n }\n}"
|
|
877
1598
|
}
|
|
878
1599
|
]
|
|
879
1600
|
},
|
|
880
1601
|
"destination-filtering": {
|
|
881
|
-
"text": "Recommended destination-mapping recipes. Drop
|
|
1602
|
+
"text": "Recommended destination-mapping recipes. Drop everything automated: botScore > 50. Drop crawlers but keep AI traffic a person triggered: botScore > 50 AND botCategory != \"ai-agent\". Keep link unfurls, which mean somebody just shared the URL: botCategory != \"link-preview\". AI visibility report: group by botProduct WHERE botCategory IN (\"ai-agent\", \"ai-crawler\"). Note that botScore is null for category \"unknown\", which means not measured rather than human, and evaluates false under a > 50 filter. The transformer never drops events; filtering is always a destination decision."
|
|
882
1603
|
},
|
|
883
1604
|
"detection-scope": {
|
|
884
|
-
"text": "
|
|
1605
|
+
"text": "Detection runs in two layers. Deterministic, first match wins: missing UA (70), AI agent or crawler UA map (90, with the category and product), non-AI crawler map for search engines, SEO tools, monitors and link unfurlers (90), isbot (80), and values impossible for the pinned context (75). Graded, capped at 60, when no deterministic rung fires: client-hint version mismatch, client hints missing on a Chromium UA, wildcard Accept on a typed context, Fetch Metadata missing or off-profile, and missing Accept-Language or Accept-Encoding. It will NOT catch: residential-proxy plus stealth Chrome with realistic behavior; in-browser agents such as Claude for Chrome, which drive a real session and are header-identical to it; UA-spoofing privacy tools. For commercial-grade detection use Cloudflare Bot Management, DataDome, or HUMAN."
|
|
885
1606
|
}
|
|
886
1607
|
}
|
|
887
1608
|
}
|