harvester_sdk 1.0.54 → 1.0.55
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/index.d.ts +160 -0
- package/dist/index.js +38 -1
- package/dist/types.d.ts +201 -0
- package/dist/types.js +36 -2
- package/index.ts +41 -0
- package/package.json +1 -1
- package/types.ts +42 -2
package/dist/index.d.ts
CHANGED
|
@@ -765,3 +765,163 @@ export declare const MongoServerReportSchema: Schema<any, import("mongoose").Mod
|
|
|
765
765
|
} & {
|
|
766
766
|
__v: number;
|
|
767
767
|
}>;
|
|
768
|
+
export declare const MongoEventSchema: Schema<any, import("mongoose").Model<any, any, any, any, any, any>, {}, {}, {}, {}, {
|
|
769
|
+
versionKey: false;
|
|
770
|
+
toJSON: {
|
|
771
|
+
virtuals: true;
|
|
772
|
+
};
|
|
773
|
+
toObject: {
|
|
774
|
+
virtuals: true;
|
|
775
|
+
};
|
|
776
|
+
}, {
|
|
777
|
+
created_at: number;
|
|
778
|
+
updated_at: number;
|
|
779
|
+
timestamp: number;
|
|
780
|
+
embeddings: number[];
|
|
781
|
+
keywords: string[];
|
|
782
|
+
type?: string | null | undefined;
|
|
783
|
+
region_id?: string | null | undefined;
|
|
784
|
+
title?: string | null | undefined;
|
|
785
|
+
geo_selection_id?: string | null | undefined;
|
|
786
|
+
geo_selection_title?: string | null | undefined;
|
|
787
|
+
data_short?: string | null | undefined;
|
|
788
|
+
data?: string | null | undefined;
|
|
789
|
+
geo_selection?: any;
|
|
790
|
+
raw_data?: import("mongoose").Types.DocumentArray<{
|
|
791
|
+
data_geo: string[];
|
|
792
|
+
platform?: "telegram" | "facebook" | "instagram" | "tiktok" | "website" | "youtube" | "twitter" | "x" | "reddit" | "linkedin" | "snapchat" | "whatsapp" | "discord" | "twitch" | null | undefined;
|
|
793
|
+
timestamp?: number | null | undefined;
|
|
794
|
+
source_id?: any;
|
|
795
|
+
data_id?: any;
|
|
796
|
+
data_url?: string | null | undefined;
|
|
797
|
+
data_text?: string | null | undefined;
|
|
798
|
+
author_username?: string | null | undefined;
|
|
799
|
+
author_id?: string | null | undefined;
|
|
800
|
+
data_region_id?: string | null | undefined;
|
|
801
|
+
}, import("mongoose").Types.Subdocument<import("mongoose").Types.ObjectId, any, {
|
|
802
|
+
data_geo: string[];
|
|
803
|
+
platform?: "telegram" | "facebook" | "instagram" | "tiktok" | "website" | "youtube" | "twitter" | "x" | "reddit" | "linkedin" | "snapchat" | "whatsapp" | "discord" | "twitch" | null | undefined;
|
|
804
|
+
timestamp?: number | null | undefined;
|
|
805
|
+
source_id?: any;
|
|
806
|
+
data_id?: any;
|
|
807
|
+
data_url?: string | null | undefined;
|
|
808
|
+
data_text?: string | null | undefined;
|
|
809
|
+
author_username?: string | null | undefined;
|
|
810
|
+
author_id?: string | null | undefined;
|
|
811
|
+
data_region_id?: string | null | undefined;
|
|
812
|
+
}> & {
|
|
813
|
+
data_geo: string[];
|
|
814
|
+
platform?: "telegram" | "facebook" | "instagram" | "tiktok" | "website" | "youtube" | "twitter" | "x" | "reddit" | "linkedin" | "snapchat" | "whatsapp" | "discord" | "twitch" | null | undefined;
|
|
815
|
+
timestamp?: number | null | undefined;
|
|
816
|
+
source_id?: any;
|
|
817
|
+
data_id?: any;
|
|
818
|
+
data_url?: string | null | undefined;
|
|
819
|
+
data_text?: string | null | undefined;
|
|
820
|
+
author_username?: string | null | undefined;
|
|
821
|
+
author_id?: string | null | undefined;
|
|
822
|
+
data_region_id?: string | null | undefined;
|
|
823
|
+
}> | null | undefined;
|
|
824
|
+
event_id?: string | null | undefined;
|
|
825
|
+
}, import("mongoose").Document<unknown, {}, import("mongoose").FlatRecord<{
|
|
826
|
+
created_at: number;
|
|
827
|
+
updated_at: number;
|
|
828
|
+
timestamp: number;
|
|
829
|
+
embeddings: number[];
|
|
830
|
+
keywords: string[];
|
|
831
|
+
type?: string | null | undefined;
|
|
832
|
+
region_id?: string | null | undefined;
|
|
833
|
+
title?: string | null | undefined;
|
|
834
|
+
geo_selection_id?: string | null | undefined;
|
|
835
|
+
geo_selection_title?: string | null | undefined;
|
|
836
|
+
data_short?: string | null | undefined;
|
|
837
|
+
data?: string | null | undefined;
|
|
838
|
+
geo_selection?: any;
|
|
839
|
+
raw_data?: import("mongoose").Types.DocumentArray<{
|
|
840
|
+
data_geo: string[];
|
|
841
|
+
platform?: "telegram" | "facebook" | "instagram" | "tiktok" | "website" | "youtube" | "twitter" | "x" | "reddit" | "linkedin" | "snapchat" | "whatsapp" | "discord" | "twitch" | null | undefined;
|
|
842
|
+
timestamp?: number | null | undefined;
|
|
843
|
+
source_id?: any;
|
|
844
|
+
data_id?: any;
|
|
845
|
+
data_url?: string | null | undefined;
|
|
846
|
+
data_text?: string | null | undefined;
|
|
847
|
+
author_username?: string | null | undefined;
|
|
848
|
+
author_id?: string | null | undefined;
|
|
849
|
+
data_region_id?: string | null | undefined;
|
|
850
|
+
}, import("mongoose").Types.Subdocument<import("mongoose").Types.ObjectId, any, {
|
|
851
|
+
data_geo: string[];
|
|
852
|
+
platform?: "telegram" | "facebook" | "instagram" | "tiktok" | "website" | "youtube" | "twitter" | "x" | "reddit" | "linkedin" | "snapchat" | "whatsapp" | "discord" | "twitch" | null | undefined;
|
|
853
|
+
timestamp?: number | null | undefined;
|
|
854
|
+
source_id?: any;
|
|
855
|
+
data_id?: any;
|
|
856
|
+
data_url?: string | null | undefined;
|
|
857
|
+
data_text?: string | null | undefined;
|
|
858
|
+
author_username?: string | null | undefined;
|
|
859
|
+
author_id?: string | null | undefined;
|
|
860
|
+
data_region_id?: string | null | undefined;
|
|
861
|
+
}> & {
|
|
862
|
+
data_geo: string[];
|
|
863
|
+
platform?: "telegram" | "facebook" | "instagram" | "tiktok" | "website" | "youtube" | "twitter" | "x" | "reddit" | "linkedin" | "snapchat" | "whatsapp" | "discord" | "twitch" | null | undefined;
|
|
864
|
+
timestamp?: number | null | undefined;
|
|
865
|
+
source_id?: any;
|
|
866
|
+
data_id?: any;
|
|
867
|
+
data_url?: string | null | undefined;
|
|
868
|
+
data_text?: string | null | undefined;
|
|
869
|
+
author_username?: string | null | undefined;
|
|
870
|
+
author_id?: string | null | undefined;
|
|
871
|
+
data_region_id?: string | null | undefined;
|
|
872
|
+
}> | null | undefined;
|
|
873
|
+
event_id?: string | null | undefined;
|
|
874
|
+
}>, {}> & import("mongoose").FlatRecord<{
|
|
875
|
+
created_at: number;
|
|
876
|
+
updated_at: number;
|
|
877
|
+
timestamp: number;
|
|
878
|
+
embeddings: number[];
|
|
879
|
+
keywords: string[];
|
|
880
|
+
type?: string | null | undefined;
|
|
881
|
+
region_id?: string | null | undefined;
|
|
882
|
+
title?: string | null | undefined;
|
|
883
|
+
geo_selection_id?: string | null | undefined;
|
|
884
|
+
geo_selection_title?: string | null | undefined;
|
|
885
|
+
data_short?: string | null | undefined;
|
|
886
|
+
data?: string | null | undefined;
|
|
887
|
+
geo_selection?: any;
|
|
888
|
+
raw_data?: import("mongoose").Types.DocumentArray<{
|
|
889
|
+
data_geo: string[];
|
|
890
|
+
platform?: "telegram" | "facebook" | "instagram" | "tiktok" | "website" | "youtube" | "twitter" | "x" | "reddit" | "linkedin" | "snapchat" | "whatsapp" | "discord" | "twitch" | null | undefined;
|
|
891
|
+
timestamp?: number | null | undefined;
|
|
892
|
+
source_id?: any;
|
|
893
|
+
data_id?: any;
|
|
894
|
+
data_url?: string | null | undefined;
|
|
895
|
+
data_text?: string | null | undefined;
|
|
896
|
+
author_username?: string | null | undefined;
|
|
897
|
+
author_id?: string | null | undefined;
|
|
898
|
+
data_region_id?: string | null | undefined;
|
|
899
|
+
}, import("mongoose").Types.Subdocument<import("mongoose").Types.ObjectId, any, {
|
|
900
|
+
data_geo: string[];
|
|
901
|
+
platform?: "telegram" | "facebook" | "instagram" | "tiktok" | "website" | "youtube" | "twitter" | "x" | "reddit" | "linkedin" | "snapchat" | "whatsapp" | "discord" | "twitch" | null | undefined;
|
|
902
|
+
timestamp?: number | null | undefined;
|
|
903
|
+
source_id?: any;
|
|
904
|
+
data_id?: any;
|
|
905
|
+
data_url?: string | null | undefined;
|
|
906
|
+
data_text?: string | null | undefined;
|
|
907
|
+
author_username?: string | null | undefined;
|
|
908
|
+
author_id?: string | null | undefined;
|
|
909
|
+
data_region_id?: string | null | undefined;
|
|
910
|
+
}> & {
|
|
911
|
+
data_geo: string[];
|
|
912
|
+
platform?: "telegram" | "facebook" | "instagram" | "tiktok" | "website" | "youtube" | "twitter" | "x" | "reddit" | "linkedin" | "snapchat" | "whatsapp" | "discord" | "twitch" | null | undefined;
|
|
913
|
+
timestamp?: number | null | undefined;
|
|
914
|
+
source_id?: any;
|
|
915
|
+
data_id?: any;
|
|
916
|
+
data_url?: string | null | undefined;
|
|
917
|
+
data_text?: string | null | undefined;
|
|
918
|
+
author_username?: string | null | undefined;
|
|
919
|
+
author_id?: string | null | undefined;
|
|
920
|
+
data_region_id?: string | null | undefined;
|
|
921
|
+
}> | null | undefined;
|
|
922
|
+
event_id?: string | null | undefined;
|
|
923
|
+
}> & {
|
|
924
|
+
_id: import("mongoose").Types.ObjectId;
|
|
925
|
+
} & {
|
|
926
|
+
__v: number;
|
|
927
|
+
}>;
|
package/dist/index.js
CHANGED
|
@@ -14,7 +14,7 @@ var __exportStar = (this && this.__exportStar) || function(m, exports) {
|
|
|
14
14
|
for (var p in m) if (p !== "default" && !Object.prototype.hasOwnProperty.call(exports, p)) __createBinding(exports, m, p);
|
|
15
15
|
};
|
|
16
16
|
Object.defineProperty(exports, "__esModule", { value: true });
|
|
17
|
-
exports.MongoServerReportSchema = exports.MongoRegionSummarySchema = exports.MongoApiKeySchema = exports.MongoGeoSelectionSchema = exports.MongoGeoSchema = exports.MongoSourceSchema = exports.MongoSourceGroupSchema = exports.MongoRegionSchema = exports.MongoDataSchema = exports.HarvesterSDKError = exports.createHarvesterSDK = exports.HarvesterSDK = void 0;
|
|
17
|
+
exports.MongoEventSchema = exports.MongoServerReportSchema = exports.MongoRegionSummarySchema = exports.MongoApiKeySchema = exports.MongoGeoSelectionSchema = exports.MongoGeoSchema = exports.MongoSourceSchema = exports.MongoSourceGroupSchema = exports.MongoRegionSchema = exports.MongoDataSchema = exports.HarvesterSDKError = exports.createHarvesterSDK = exports.HarvesterSDK = void 0;
|
|
18
18
|
// Export types for consumers
|
|
19
19
|
__exportStar(require("./types"), exports);
|
|
20
20
|
// Export SDK
|
|
@@ -341,3 +341,40 @@ exports.MongoServerReportSchema = new mongoose_1.Schema({
|
|
|
341
341
|
toJSON: { virtuals: true },
|
|
342
342
|
toObject: { virtuals: true },
|
|
343
343
|
});
|
|
344
|
+
exports.MongoEventSchema = new mongoose_1.Schema({
|
|
345
|
+
event_id: { type: String },
|
|
346
|
+
timestamp: { type: Number, default: Date.now },
|
|
347
|
+
region_id: { type: String },
|
|
348
|
+
geo_selection_id: { type: String },
|
|
349
|
+
geo_selection_title: { type: String },
|
|
350
|
+
type: { type: String },
|
|
351
|
+
title: { type: String },
|
|
352
|
+
data_short: { type: String },
|
|
353
|
+
data: { type: String },
|
|
354
|
+
created_at: { type: Number, default: Date.now },
|
|
355
|
+
updated_at: { type: Number, default: Date.now },
|
|
356
|
+
keywords: { type: [String], default: [] },
|
|
357
|
+
geo_selection: { type: mongoose_1.Schema.Types.Mixed },
|
|
358
|
+
raw_data: {
|
|
359
|
+
type: [
|
|
360
|
+
{
|
|
361
|
+
platform: { type: String, enum: types_1.platformsList },
|
|
362
|
+
source_id: { type: mongoose_1.Schema.Types.Mixed },
|
|
363
|
+
data_id: { type: mongoose_1.Schema.Types.Mixed },
|
|
364
|
+
data_url: { type: String },
|
|
365
|
+
data_text: { type: String },
|
|
366
|
+
author_username: { type: String },
|
|
367
|
+
author_id: { type: String },
|
|
368
|
+
timestamp: { type: Number },
|
|
369
|
+
data_geo: { type: [String], default: [] },
|
|
370
|
+
data_region_id: { type: String },
|
|
371
|
+
},
|
|
372
|
+
],
|
|
373
|
+
default: undefined,
|
|
374
|
+
},
|
|
375
|
+
embeddings: { type: [Number], default: [] }, // vector embeddings for ML tasks
|
|
376
|
+
}, {
|
|
377
|
+
versionKey: false,
|
|
378
|
+
toJSON: { virtuals: true },
|
|
379
|
+
toObject: { virtuals: true },
|
|
380
|
+
});
|
package/dist/types.d.ts
CHANGED
|
@@ -2091,3 +2091,204 @@ export type FacebookMetadataType = z.infer<typeof facebookMetadataSchema>;
|
|
|
2091
2091
|
export type InstagramMetadataType = z.infer<typeof instagramMetadataSchema>;
|
|
2092
2092
|
export type TiktokMetadataType = z.infer<typeof tiktokMetadataSchema>;
|
|
2093
2093
|
export type WebsiteMetadataType = z.infer<typeof websiteMetadataSchema>;
|
|
2094
|
+
/**
|
|
2095
|
+
* Harvester events
|
|
2096
|
+
*/
|
|
2097
|
+
export declare const zodEventSchema: z.ZodObject<{
|
|
2098
|
+
_id: z.ZodOptional<z.ZodString>;
|
|
2099
|
+
region_id: z.ZodString;
|
|
2100
|
+
geo_selection_id: z.ZodOptional<z.ZodString>;
|
|
2101
|
+
geo_selection_title: z.ZodOptional<z.ZodString>;
|
|
2102
|
+
type: z.ZodString;
|
|
2103
|
+
title: z.ZodString;
|
|
2104
|
+
data_short: z.ZodOptional<z.ZodString>;
|
|
2105
|
+
data: z.ZodString;
|
|
2106
|
+
created_at: z.ZodNumber;
|
|
2107
|
+
updated_at: z.ZodNumber;
|
|
2108
|
+
keywords: z.ZodOptional<z.ZodArray<z.ZodString, "many">>;
|
|
2109
|
+
geo_selection: z.ZodObject<{
|
|
2110
|
+
_id: z.ZodOptional<z.ZodString>;
|
|
2111
|
+
title: z.ZodOptional<z.ZodString>;
|
|
2112
|
+
description: z.ZodOptional<z.ZodString>;
|
|
2113
|
+
type: z.ZodOptional<z.ZodString>;
|
|
2114
|
+
geos: z.ZodOptional<z.ZodArray<z.ZodString, "many">>;
|
|
2115
|
+
region_id: z.ZodOptional<z.ZodString>;
|
|
2116
|
+
notes: z.ZodOptional<z.ZodString>;
|
|
2117
|
+
status: z.ZodOptional<z.ZodEnum<["active", "inactive", "deleted"]>>;
|
|
2118
|
+
created_at: z.ZodOptional<z.ZodNumber>;
|
|
2119
|
+
updated_at: z.ZodOptional<z.ZodNumber>;
|
|
2120
|
+
coordinates: z.ZodOptional<z.ZodObject<{
|
|
2121
|
+
type: z.ZodOptional<z.ZodEnum<["point", "polygon"]>>;
|
|
2122
|
+
coordinates: z.ZodOptional<z.ZodUnion<[z.ZodTuple<[z.ZodNumber, z.ZodNumber], null>, z.ZodArray<z.ZodTuple<[z.ZodNumber, z.ZodNumber], null>, "many">]>>;
|
|
2123
|
+
}, "strip", z.ZodTypeAny, {
|
|
2124
|
+
type?: "point" | "polygon" | undefined;
|
|
2125
|
+
coordinates?: [number, number] | [number, number][] | undefined;
|
|
2126
|
+
}, {
|
|
2127
|
+
type?: "point" | "polygon" | undefined;
|
|
2128
|
+
coordinates?: [number, number] | [number, number][] | undefined;
|
|
2129
|
+
}>>;
|
|
2130
|
+
lat: z.ZodOptional<z.ZodNumber>;
|
|
2131
|
+
lng: z.ZodOptional<z.ZodNumber>;
|
|
2132
|
+
}, "strip", z.ZodTypeAny, {
|
|
2133
|
+
_id?: string | undefined;
|
|
2134
|
+
created_at?: number | undefined;
|
|
2135
|
+
updated_at?: number | undefined;
|
|
2136
|
+
status?: "active" | "inactive" | "deleted" | undefined;
|
|
2137
|
+
notes?: string | undefined;
|
|
2138
|
+
type?: string | undefined;
|
|
2139
|
+
coordinates?: {
|
|
2140
|
+
type?: "point" | "polygon" | undefined;
|
|
2141
|
+
coordinates?: [number, number] | [number, number][] | undefined;
|
|
2142
|
+
} | undefined;
|
|
2143
|
+
description?: string | undefined;
|
|
2144
|
+
region_id?: string | undefined;
|
|
2145
|
+
title?: string | undefined;
|
|
2146
|
+
geos?: string[] | undefined;
|
|
2147
|
+
lat?: number | undefined;
|
|
2148
|
+
lng?: number | undefined;
|
|
2149
|
+
}, {
|
|
2150
|
+
_id?: string | undefined;
|
|
2151
|
+
created_at?: number | undefined;
|
|
2152
|
+
updated_at?: number | undefined;
|
|
2153
|
+
status?: "active" | "inactive" | "deleted" | undefined;
|
|
2154
|
+
notes?: string | undefined;
|
|
2155
|
+
type?: string | undefined;
|
|
2156
|
+
coordinates?: {
|
|
2157
|
+
type?: "point" | "polygon" | undefined;
|
|
2158
|
+
coordinates?: [number, number] | [number, number][] | undefined;
|
|
2159
|
+
} | undefined;
|
|
2160
|
+
description?: string | undefined;
|
|
2161
|
+
region_id?: string | undefined;
|
|
2162
|
+
title?: string | undefined;
|
|
2163
|
+
geos?: string[] | undefined;
|
|
2164
|
+
lat?: number | undefined;
|
|
2165
|
+
lng?: number | undefined;
|
|
2166
|
+
}>;
|
|
2167
|
+
raw_data: z.ZodOptional<z.ZodArray<z.ZodObject<{
|
|
2168
|
+
_id: z.ZodOptional<z.ZodString>;
|
|
2169
|
+
platform: z.ZodEnum<["telegram", "facebook", "instagram", "tiktok", "website", "youtube", "twitter", "x", "reddit", "linkedin", "snapchat", "whatsapp", "discord", "twitch"]>;
|
|
2170
|
+
source_id: z.ZodUnion<[z.ZodString, z.ZodNumber]>;
|
|
2171
|
+
data_id: z.ZodUnion<[z.ZodString, z.ZodNumber]>;
|
|
2172
|
+
data_url: z.ZodOptional<z.ZodString>;
|
|
2173
|
+
data_text: z.ZodOptional<z.ZodString>;
|
|
2174
|
+
author_username: z.ZodOptional<z.ZodString>;
|
|
2175
|
+
author_id: z.ZodOptional<z.ZodString>;
|
|
2176
|
+
timestamp: z.ZodOptional<z.ZodNumber>;
|
|
2177
|
+
data_geo: z.ZodDefault<z.ZodOptional<z.ZodArray<z.ZodString, "many">>>;
|
|
2178
|
+
data_region_id: z.ZodOptional<z.ZodString>;
|
|
2179
|
+
}, "strip", z.ZodTypeAny, {
|
|
2180
|
+
platform: "telegram" | "facebook" | "instagram" | "tiktok" | "website" | "youtube" | "twitter" | "x" | "reddit" | "linkedin" | "snapchat" | "whatsapp" | "discord" | "twitch";
|
|
2181
|
+
source_id: string | number;
|
|
2182
|
+
data_id: string | number;
|
|
2183
|
+
data_geo: string[];
|
|
2184
|
+
_id?: string | undefined;
|
|
2185
|
+
timestamp?: number | undefined;
|
|
2186
|
+
data_url?: string | undefined;
|
|
2187
|
+
data_text?: string | undefined;
|
|
2188
|
+
author_username?: string | undefined;
|
|
2189
|
+
author_id?: string | undefined;
|
|
2190
|
+
data_region_id?: string | undefined;
|
|
2191
|
+
}, {
|
|
2192
|
+
platform: "telegram" | "facebook" | "instagram" | "tiktok" | "website" | "youtube" | "twitter" | "x" | "reddit" | "linkedin" | "snapchat" | "whatsapp" | "discord" | "twitch";
|
|
2193
|
+
source_id: string | number;
|
|
2194
|
+
data_id: string | number;
|
|
2195
|
+
_id?: string | undefined;
|
|
2196
|
+
timestamp?: number | undefined;
|
|
2197
|
+
data_url?: string | undefined;
|
|
2198
|
+
data_text?: string | undefined;
|
|
2199
|
+
data_geo?: string[] | undefined;
|
|
2200
|
+
author_username?: string | undefined;
|
|
2201
|
+
author_id?: string | undefined;
|
|
2202
|
+
data_region_id?: string | undefined;
|
|
2203
|
+
}>, "many">>;
|
|
2204
|
+
embeddings: z.ZodOptional<z.ZodArray<z.ZodNumber, "many">>;
|
|
2205
|
+
}, "strip", z.ZodTypeAny, {
|
|
2206
|
+
created_at: number;
|
|
2207
|
+
updated_at: number;
|
|
2208
|
+
type: string;
|
|
2209
|
+
region_id: string;
|
|
2210
|
+
title: string;
|
|
2211
|
+
data: string;
|
|
2212
|
+
geo_selection: {
|
|
2213
|
+
_id?: string | undefined;
|
|
2214
|
+
created_at?: number | undefined;
|
|
2215
|
+
updated_at?: number | undefined;
|
|
2216
|
+
status?: "active" | "inactive" | "deleted" | undefined;
|
|
2217
|
+
notes?: string | undefined;
|
|
2218
|
+
type?: string | undefined;
|
|
2219
|
+
coordinates?: {
|
|
2220
|
+
type?: "point" | "polygon" | undefined;
|
|
2221
|
+
coordinates?: [number, number] | [number, number][] | undefined;
|
|
2222
|
+
} | undefined;
|
|
2223
|
+
description?: string | undefined;
|
|
2224
|
+
region_id?: string | undefined;
|
|
2225
|
+
title?: string | undefined;
|
|
2226
|
+
geos?: string[] | undefined;
|
|
2227
|
+
lat?: number | undefined;
|
|
2228
|
+
lng?: number | undefined;
|
|
2229
|
+
};
|
|
2230
|
+
_id?: string | undefined;
|
|
2231
|
+
embeddings?: number[] | undefined;
|
|
2232
|
+
keywords?: string[] | undefined;
|
|
2233
|
+
geo_selection_id?: string | undefined;
|
|
2234
|
+
geo_selection_title?: string | undefined;
|
|
2235
|
+
data_short?: string | undefined;
|
|
2236
|
+
raw_data?: {
|
|
2237
|
+
platform: "telegram" | "facebook" | "instagram" | "tiktok" | "website" | "youtube" | "twitter" | "x" | "reddit" | "linkedin" | "snapchat" | "whatsapp" | "discord" | "twitch";
|
|
2238
|
+
source_id: string | number;
|
|
2239
|
+
data_id: string | number;
|
|
2240
|
+
data_geo: string[];
|
|
2241
|
+
_id?: string | undefined;
|
|
2242
|
+
timestamp?: number | undefined;
|
|
2243
|
+
data_url?: string | undefined;
|
|
2244
|
+
data_text?: string | undefined;
|
|
2245
|
+
author_username?: string | undefined;
|
|
2246
|
+
author_id?: string | undefined;
|
|
2247
|
+
data_region_id?: string | undefined;
|
|
2248
|
+
}[] | undefined;
|
|
2249
|
+
}, {
|
|
2250
|
+
created_at: number;
|
|
2251
|
+
updated_at: number;
|
|
2252
|
+
type: string;
|
|
2253
|
+
region_id: string;
|
|
2254
|
+
title: string;
|
|
2255
|
+
data: string;
|
|
2256
|
+
geo_selection: {
|
|
2257
|
+
_id?: string | undefined;
|
|
2258
|
+
created_at?: number | undefined;
|
|
2259
|
+
updated_at?: number | undefined;
|
|
2260
|
+
status?: "active" | "inactive" | "deleted" | undefined;
|
|
2261
|
+
notes?: string | undefined;
|
|
2262
|
+
type?: string | undefined;
|
|
2263
|
+
coordinates?: {
|
|
2264
|
+
type?: "point" | "polygon" | undefined;
|
|
2265
|
+
coordinates?: [number, number] | [number, number][] | undefined;
|
|
2266
|
+
} | undefined;
|
|
2267
|
+
description?: string | undefined;
|
|
2268
|
+
region_id?: string | undefined;
|
|
2269
|
+
title?: string | undefined;
|
|
2270
|
+
geos?: string[] | undefined;
|
|
2271
|
+
lat?: number | undefined;
|
|
2272
|
+
lng?: number | undefined;
|
|
2273
|
+
};
|
|
2274
|
+
_id?: string | undefined;
|
|
2275
|
+
embeddings?: number[] | undefined;
|
|
2276
|
+
keywords?: string[] | undefined;
|
|
2277
|
+
geo_selection_id?: string | undefined;
|
|
2278
|
+
geo_selection_title?: string | undefined;
|
|
2279
|
+
data_short?: string | undefined;
|
|
2280
|
+
raw_data?: {
|
|
2281
|
+
platform: "telegram" | "facebook" | "instagram" | "tiktok" | "website" | "youtube" | "twitter" | "x" | "reddit" | "linkedin" | "snapchat" | "whatsapp" | "discord" | "twitch";
|
|
2282
|
+
source_id: string | number;
|
|
2283
|
+
data_id: string | number;
|
|
2284
|
+
_id?: string | undefined;
|
|
2285
|
+
timestamp?: number | undefined;
|
|
2286
|
+
data_url?: string | undefined;
|
|
2287
|
+
data_text?: string | undefined;
|
|
2288
|
+
data_geo?: string[] | undefined;
|
|
2289
|
+
author_username?: string | undefined;
|
|
2290
|
+
author_id?: string | undefined;
|
|
2291
|
+
data_region_id?: string | undefined;
|
|
2292
|
+
}[] | undefined;
|
|
2293
|
+
}>;
|
|
2294
|
+
export type EventType = z.infer<typeof zodEventSchema>;
|
package/dist/types.js
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
"use strict";
|
|
2
2
|
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
|
-
exports.zodServerReportSchema = exports.zodRegionSummarySchema = exports.zodDashboardStatsSchema = exports.zodDataSchemaWithValidation = exports.zodDataSchema = exports.authorSchema = exports.repliesInfoSchema = exports.mediaItemSchema = exports.websiteMetadataSchema = exports.tiktokMetadataSchema = exports.instagramMetadataSchema = exports.facebookMetadataSchema = exports.telegramMetadataSchema = exports.zodGeoSelectionSchema = exports.zodGeoSchema = exports.zodSourceSchema = exports.zodSourceGroupSchema = exports.zodRegionSchema = exports.generalStatusList = exports.sourceStatusList = exports.priorityLevels = exports.platformEntityMap = exports.entityTypesList = exports.platformsList = void 0;
|
|
3
|
+
exports.zodEventSchema = exports.zodServerReportSchema = exports.zodRegionSummarySchema = exports.zodDashboardStatsSchema = exports.zodDataSchemaWithValidation = exports.zodDataSchema = exports.authorSchema = exports.repliesInfoSchema = exports.mediaItemSchema = exports.websiteMetadataSchema = exports.tiktokMetadataSchema = exports.instagramMetadataSchema = exports.facebookMetadataSchema = exports.telegramMetadataSchema = exports.zodGeoSelectionSchema = exports.zodGeoSchema = exports.zodSourceSchema = exports.zodSourceGroupSchema = exports.zodRegionSchema = exports.generalStatusList = exports.sourceStatusList = exports.priorityLevels = exports.platformEntityMap = exports.entityTypesList = exports.platformsList = void 0;
|
|
4
4
|
const zod_1 = require("zod");
|
|
5
5
|
exports.platformsList = [
|
|
6
6
|
'telegram',
|
|
@@ -25,7 +25,7 @@ exports.entityTypesList = [
|
|
|
25
25
|
'channel',
|
|
26
26
|
'hashtag',
|
|
27
27
|
'website',
|
|
28
|
-
'rss'
|
|
28
|
+
'rss',
|
|
29
29
|
];
|
|
30
30
|
// Platform-specific entity mappings
|
|
31
31
|
exports.platformEntityMap = {
|
|
@@ -418,3 +418,37 @@ exports.zodServerReportSchema = zod_1.z.object({
|
|
|
418
418
|
})
|
|
419
419
|
.optional(),
|
|
420
420
|
});
|
|
421
|
+
// -------------------
|
|
422
|
+
/**
|
|
423
|
+
* Harvester events
|
|
424
|
+
*/
|
|
425
|
+
exports.zodEventSchema = zod_1.z.object({
|
|
426
|
+
_id: zod_1.z.string().optional(),
|
|
427
|
+
region_id: zod_1.z.string(),
|
|
428
|
+
geo_selection_id: zod_1.z.string().optional(),
|
|
429
|
+
geo_selection_title: zod_1.z.string().optional(),
|
|
430
|
+
type: zod_1.z.string(),
|
|
431
|
+
title: zod_1.z.string(),
|
|
432
|
+
data_short: zod_1.z.string().optional(),
|
|
433
|
+
data: zod_1.z.string(),
|
|
434
|
+
created_at: zod_1.z.number(),
|
|
435
|
+
updated_at: zod_1.z.number(),
|
|
436
|
+
keywords: zod_1.z.array(zod_1.z.string()).optional(),
|
|
437
|
+
geo_selection: exports.zodGeoSelectionSchema,
|
|
438
|
+
raw_data: zod_1.z
|
|
439
|
+
.array(zod_1.z.object({
|
|
440
|
+
_id: zod_1.z.string().optional(),
|
|
441
|
+
platform: zod_1.z.enum(exports.platformsList),
|
|
442
|
+
source_id: zod_1.z.string().or(zod_1.z.number()),
|
|
443
|
+
data_id: zod_1.z.string().or(zod_1.z.number()),
|
|
444
|
+
data_url: zod_1.z.string().url().optional(),
|
|
445
|
+
data_text: zod_1.z.string().optional(),
|
|
446
|
+
author_username: zod_1.z.string().optional(),
|
|
447
|
+
author_id: zod_1.z.string().optional(),
|
|
448
|
+
timestamp: zod_1.z.number().optional(),
|
|
449
|
+
data_geo: zod_1.z.array(zod_1.z.string()).optional().default([]),
|
|
450
|
+
data_region_id: zod_1.z.string().optional(),
|
|
451
|
+
}))
|
|
452
|
+
.optional(),
|
|
453
|
+
embeddings: zod_1.z.array(zod_1.z.number()).optional(), // e.g., vector embeddings for ML tasks
|
|
454
|
+
});
|
package/index.ts
CHANGED
|
@@ -377,3 +377,44 @@ export const MongoServerReportSchema = new Schema(
|
|
|
377
377
|
toObject: { virtuals: true },
|
|
378
378
|
},
|
|
379
379
|
);
|
|
380
|
+
|
|
381
|
+
export const MongoEventSchema = new Schema(
|
|
382
|
+
{
|
|
383
|
+
event_id: { type: String }, // unique identifier for the event
|
|
384
|
+
timestamp: { type: Number, default: Date.now }, // event creation timestamp
|
|
385
|
+
region_id: { type: String }, // e.g., region ID
|
|
386
|
+
geo_selection_id: { type: String }, // e.g., geo selection ID
|
|
387
|
+
geo_selection_title: { type: String }, // e.g., geo selection title
|
|
388
|
+
type: { type: String }, // e.g., 'alert', 'info'
|
|
389
|
+
title: { type: String }, // e.g., 'Red sea escalating'
|
|
390
|
+
data_short: { type: String }, // e.g., 'overview of the event'
|
|
391
|
+
data: { type: String }, // in depth data about the event, this string is updated each time the object is updated
|
|
392
|
+
created_at: { type: Number, default: Date.now }, // creation date
|
|
393
|
+
updated_at: { type: Number, default: Date.now }, // last update date
|
|
394
|
+
keywords: { type: [String], default: [] }, // e.g., ['red sea', 'escalating']
|
|
395
|
+
geo_selection: { type: Schema.Types.Mixed }, // embedded geo selection object
|
|
396
|
+
raw_data: {
|
|
397
|
+
type: [
|
|
398
|
+
{
|
|
399
|
+
platform: { type: String, enum: platformsList },
|
|
400
|
+
source_id: { type: Schema.Types.Mixed },
|
|
401
|
+
data_id: { type: Schema.Types.Mixed },
|
|
402
|
+
data_url: { type: String },
|
|
403
|
+
data_text: { type: String },
|
|
404
|
+
author_username: { type: String },
|
|
405
|
+
author_id: { type: String },
|
|
406
|
+
timestamp: { type: Number },
|
|
407
|
+
data_geo: { type: [String], default: [] },
|
|
408
|
+
data_region_id: { type: String },
|
|
409
|
+
},
|
|
410
|
+
],
|
|
411
|
+
default: undefined,
|
|
412
|
+
}, // raw data items that triggered the event
|
|
413
|
+
embeddings: { type: [Number], default: [] }, // vector embeddings for ML tasks
|
|
414
|
+
},
|
|
415
|
+
{
|
|
416
|
+
versionKey: false,
|
|
417
|
+
toJSON: { virtuals: true },
|
|
418
|
+
toObject: { virtuals: true },
|
|
419
|
+
},
|
|
420
|
+
);
|
package/package.json
CHANGED
package/types.ts
CHANGED
|
@@ -24,7 +24,7 @@ export const entityTypesList = [
|
|
|
24
24
|
'channel',
|
|
25
25
|
'hashtag',
|
|
26
26
|
'website',
|
|
27
|
-
'rss'
|
|
27
|
+
'rss',
|
|
28
28
|
] as const;
|
|
29
29
|
|
|
30
30
|
// Platform-specific entity mappings
|
|
@@ -97,7 +97,7 @@ export const zodSourceGroupSchema = z.object({
|
|
|
97
97
|
error_count: z.number().optional(),
|
|
98
98
|
total_posts: z.number().optional(),
|
|
99
99
|
sources_discovered: z.number().optional(),
|
|
100
|
-
})
|
|
100
|
+
}),
|
|
101
101
|
)
|
|
102
102
|
.optional(),
|
|
103
103
|
});
|
|
@@ -482,3 +482,43 @@ export type FacebookMetadataType = z.infer<typeof facebookMetadataSchema>;
|
|
|
482
482
|
export type InstagramMetadataType = z.infer<typeof instagramMetadataSchema>;
|
|
483
483
|
export type TiktokMetadataType = z.infer<typeof tiktokMetadataSchema>;
|
|
484
484
|
export type WebsiteMetadataType = z.infer<typeof websiteMetadataSchema>;
|
|
485
|
+
|
|
486
|
+
// -------------------
|
|
487
|
+
/**
|
|
488
|
+
* Harvester events
|
|
489
|
+
*/
|
|
490
|
+
|
|
491
|
+
export const zodEventSchema = z.object({
|
|
492
|
+
_id: z.string().optional(),
|
|
493
|
+
region_id: z.string(), // e.g., region ID
|
|
494
|
+
geo_selection_id: z.string().optional(), // e.g., geo selection ID
|
|
495
|
+
geo_selection_title: z.string().optional(), // e.g., geo selection title
|
|
496
|
+
type: z.string(), // e.g., 'alert', 'info'
|
|
497
|
+
title: z.string(), // e.g., 'Red sea escalating'
|
|
498
|
+
data_short: z.string().optional(), // e.g., 'overview of the event'
|
|
499
|
+
data: z.string(), // in depth data about the event, this string is updated each time the object is updated
|
|
500
|
+
created_at: z.number(), // e.g., 1751210833000
|
|
501
|
+
updated_at: z.number(), // e.g., 1751210833000
|
|
502
|
+
keywords: z.array(z.string()).optional(), // e.g., ['red sea', 'escalating']
|
|
503
|
+
geo_selection: zodGeoSelectionSchema,
|
|
504
|
+
raw_data: z
|
|
505
|
+
.array(
|
|
506
|
+
z.object({
|
|
507
|
+
_id: z.string().optional(),
|
|
508
|
+
platform: z.enum(platformsList),
|
|
509
|
+
source_id: z.string().or(z.number()),
|
|
510
|
+
data_id: z.string().or(z.number()),
|
|
511
|
+
data_url: z.string().url().optional(),
|
|
512
|
+
data_text: z.string().optional(),
|
|
513
|
+
author_username: z.string().optional(),
|
|
514
|
+
author_id: z.string().optional(),
|
|
515
|
+
timestamp: z.number().optional(),
|
|
516
|
+
data_geo: z.array(z.string()).optional().default([]),
|
|
517
|
+
data_region_id: z.string().optional(),
|
|
518
|
+
}),
|
|
519
|
+
)
|
|
520
|
+
.optional(), // raw data items that triggered the event
|
|
521
|
+
embeddings: z.array(z.number()).optional(), // e.g., vector embeddings for ML tasks
|
|
522
|
+
});
|
|
523
|
+
|
|
524
|
+
export type EventType = z.infer<typeof zodEventSchema>;
|