harvester_sdk 1.0.53 → 1.0.55

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (7) hide show
  1. package/dist/index.d.ts +160 -0
  2. package/dist/index.js +38 -1
  3. package/dist/types.d.ts +206 -5
  4. package/dist/types.js +37 -2
  5. package/index.ts +41 -1
  6. package/package.json +1 -1
  7. package/types.ts +43 -2
package/dist/index.d.ts CHANGED
@@ -765,3 +765,163 @@ export declare const MongoServerReportSchema: Schema<any, import("mongoose").Mod
765
765
  } & {
766
766
  __v: number;
767
767
  }>;
768
+ export declare const MongoEventSchema: Schema<any, import("mongoose").Model<any, any, any, any, any, any>, {}, {}, {}, {}, {
769
+ versionKey: false;
770
+ toJSON: {
771
+ virtuals: true;
772
+ };
773
+ toObject: {
774
+ virtuals: true;
775
+ };
776
+ }, {
777
+ created_at: number;
778
+ updated_at: number;
779
+ timestamp: number;
780
+ embeddings: number[];
781
+ keywords: string[];
782
+ type?: string | null | undefined;
783
+ region_id?: string | null | undefined;
784
+ title?: string | null | undefined;
785
+ geo_selection_id?: string | null | undefined;
786
+ geo_selection_title?: string | null | undefined;
787
+ data_short?: string | null | undefined;
788
+ data?: string | null | undefined;
789
+ geo_selection?: any;
790
+ raw_data?: import("mongoose").Types.DocumentArray<{
791
+ data_geo: string[];
792
+ platform?: "telegram" | "facebook" | "instagram" | "tiktok" | "website" | "youtube" | "twitter" | "x" | "reddit" | "linkedin" | "snapchat" | "whatsapp" | "discord" | "twitch" | null | undefined;
793
+ timestamp?: number | null | undefined;
794
+ source_id?: any;
795
+ data_id?: any;
796
+ data_url?: string | null | undefined;
797
+ data_text?: string | null | undefined;
798
+ author_username?: string | null | undefined;
799
+ author_id?: string | null | undefined;
800
+ data_region_id?: string | null | undefined;
801
+ }, import("mongoose").Types.Subdocument<import("mongoose").Types.ObjectId, any, {
802
+ data_geo: string[];
803
+ platform?: "telegram" | "facebook" | "instagram" | "tiktok" | "website" | "youtube" | "twitter" | "x" | "reddit" | "linkedin" | "snapchat" | "whatsapp" | "discord" | "twitch" | null | undefined;
804
+ timestamp?: number | null | undefined;
805
+ source_id?: any;
806
+ data_id?: any;
807
+ data_url?: string | null | undefined;
808
+ data_text?: string | null | undefined;
809
+ author_username?: string | null | undefined;
810
+ author_id?: string | null | undefined;
811
+ data_region_id?: string | null | undefined;
812
+ }> & {
813
+ data_geo: string[];
814
+ platform?: "telegram" | "facebook" | "instagram" | "tiktok" | "website" | "youtube" | "twitter" | "x" | "reddit" | "linkedin" | "snapchat" | "whatsapp" | "discord" | "twitch" | null | undefined;
815
+ timestamp?: number | null | undefined;
816
+ source_id?: any;
817
+ data_id?: any;
818
+ data_url?: string | null | undefined;
819
+ data_text?: string | null | undefined;
820
+ author_username?: string | null | undefined;
821
+ author_id?: string | null | undefined;
822
+ data_region_id?: string | null | undefined;
823
+ }> | null | undefined;
824
+ event_id?: string | null | undefined;
825
+ }, import("mongoose").Document<unknown, {}, import("mongoose").FlatRecord<{
826
+ created_at: number;
827
+ updated_at: number;
828
+ timestamp: number;
829
+ embeddings: number[];
830
+ keywords: string[];
831
+ type?: string | null | undefined;
832
+ region_id?: string | null | undefined;
833
+ title?: string | null | undefined;
834
+ geo_selection_id?: string | null | undefined;
835
+ geo_selection_title?: string | null | undefined;
836
+ data_short?: string | null | undefined;
837
+ data?: string | null | undefined;
838
+ geo_selection?: any;
839
+ raw_data?: import("mongoose").Types.DocumentArray<{
840
+ data_geo: string[];
841
+ platform?: "telegram" | "facebook" | "instagram" | "tiktok" | "website" | "youtube" | "twitter" | "x" | "reddit" | "linkedin" | "snapchat" | "whatsapp" | "discord" | "twitch" | null | undefined;
842
+ timestamp?: number | null | undefined;
843
+ source_id?: any;
844
+ data_id?: any;
845
+ data_url?: string | null | undefined;
846
+ data_text?: string | null | undefined;
847
+ author_username?: string | null | undefined;
848
+ author_id?: string | null | undefined;
849
+ data_region_id?: string | null | undefined;
850
+ }, import("mongoose").Types.Subdocument<import("mongoose").Types.ObjectId, any, {
851
+ data_geo: string[];
852
+ platform?: "telegram" | "facebook" | "instagram" | "tiktok" | "website" | "youtube" | "twitter" | "x" | "reddit" | "linkedin" | "snapchat" | "whatsapp" | "discord" | "twitch" | null | undefined;
853
+ timestamp?: number | null | undefined;
854
+ source_id?: any;
855
+ data_id?: any;
856
+ data_url?: string | null | undefined;
857
+ data_text?: string | null | undefined;
858
+ author_username?: string | null | undefined;
859
+ author_id?: string | null | undefined;
860
+ data_region_id?: string | null | undefined;
861
+ }> & {
862
+ data_geo: string[];
863
+ platform?: "telegram" | "facebook" | "instagram" | "tiktok" | "website" | "youtube" | "twitter" | "x" | "reddit" | "linkedin" | "snapchat" | "whatsapp" | "discord" | "twitch" | null | undefined;
864
+ timestamp?: number | null | undefined;
865
+ source_id?: any;
866
+ data_id?: any;
867
+ data_url?: string | null | undefined;
868
+ data_text?: string | null | undefined;
869
+ author_username?: string | null | undefined;
870
+ author_id?: string | null | undefined;
871
+ data_region_id?: string | null | undefined;
872
+ }> | null | undefined;
873
+ event_id?: string | null | undefined;
874
+ }>, {}> & import("mongoose").FlatRecord<{
875
+ created_at: number;
876
+ updated_at: number;
877
+ timestamp: number;
878
+ embeddings: number[];
879
+ keywords: string[];
880
+ type?: string | null | undefined;
881
+ region_id?: string | null | undefined;
882
+ title?: string | null | undefined;
883
+ geo_selection_id?: string | null | undefined;
884
+ geo_selection_title?: string | null | undefined;
885
+ data_short?: string | null | undefined;
886
+ data?: string | null | undefined;
887
+ geo_selection?: any;
888
+ raw_data?: import("mongoose").Types.DocumentArray<{
889
+ data_geo: string[];
890
+ platform?: "telegram" | "facebook" | "instagram" | "tiktok" | "website" | "youtube" | "twitter" | "x" | "reddit" | "linkedin" | "snapchat" | "whatsapp" | "discord" | "twitch" | null | undefined;
891
+ timestamp?: number | null | undefined;
892
+ source_id?: any;
893
+ data_id?: any;
894
+ data_url?: string | null | undefined;
895
+ data_text?: string | null | undefined;
896
+ author_username?: string | null | undefined;
897
+ author_id?: string | null | undefined;
898
+ data_region_id?: string | null | undefined;
899
+ }, import("mongoose").Types.Subdocument<import("mongoose").Types.ObjectId, any, {
900
+ data_geo: string[];
901
+ platform?: "telegram" | "facebook" | "instagram" | "tiktok" | "website" | "youtube" | "twitter" | "x" | "reddit" | "linkedin" | "snapchat" | "whatsapp" | "discord" | "twitch" | null | undefined;
902
+ timestamp?: number | null | undefined;
903
+ source_id?: any;
904
+ data_id?: any;
905
+ data_url?: string | null | undefined;
906
+ data_text?: string | null | undefined;
907
+ author_username?: string | null | undefined;
908
+ author_id?: string | null | undefined;
909
+ data_region_id?: string | null | undefined;
910
+ }> & {
911
+ data_geo: string[];
912
+ platform?: "telegram" | "facebook" | "instagram" | "tiktok" | "website" | "youtube" | "twitter" | "x" | "reddit" | "linkedin" | "snapchat" | "whatsapp" | "discord" | "twitch" | null | undefined;
913
+ timestamp?: number | null | undefined;
914
+ source_id?: any;
915
+ data_id?: any;
916
+ data_url?: string | null | undefined;
917
+ data_text?: string | null | undefined;
918
+ author_username?: string | null | undefined;
919
+ author_id?: string | null | undefined;
920
+ data_region_id?: string | null | undefined;
921
+ }> | null | undefined;
922
+ event_id?: string | null | undefined;
923
+ }> & {
924
+ _id: import("mongoose").Types.ObjectId;
925
+ } & {
926
+ __v: number;
927
+ }>;
package/dist/index.js CHANGED
@@ -14,7 +14,7 @@ var __exportStar = (this && this.__exportStar) || function(m, exports) {
14
14
  for (var p in m) if (p !== "default" && !Object.prototype.hasOwnProperty.call(exports, p)) __createBinding(exports, m, p);
15
15
  };
16
16
  Object.defineProperty(exports, "__esModule", { value: true });
17
- exports.MongoServerReportSchema = exports.MongoRegionSummarySchema = exports.MongoApiKeySchema = exports.MongoGeoSelectionSchema = exports.MongoGeoSchema = exports.MongoSourceSchema = exports.MongoSourceGroupSchema = exports.MongoRegionSchema = exports.MongoDataSchema = exports.HarvesterSDKError = exports.createHarvesterSDK = exports.HarvesterSDK = void 0;
17
+ exports.MongoEventSchema = exports.MongoServerReportSchema = exports.MongoRegionSummarySchema = exports.MongoApiKeySchema = exports.MongoGeoSelectionSchema = exports.MongoGeoSchema = exports.MongoSourceSchema = exports.MongoSourceGroupSchema = exports.MongoRegionSchema = exports.MongoDataSchema = exports.HarvesterSDKError = exports.createHarvesterSDK = exports.HarvesterSDK = void 0;
18
18
  // Export types for consumers
19
19
  __exportStar(require("./types"), exports);
20
20
  // Export SDK
@@ -341,3 +341,40 @@ exports.MongoServerReportSchema = new mongoose_1.Schema({
341
341
  toJSON: { virtuals: true },
342
342
  toObject: { virtuals: true },
343
343
  });
344
+ exports.MongoEventSchema = new mongoose_1.Schema({
345
+ event_id: { type: String },
346
+ timestamp: { type: Number, default: Date.now },
347
+ region_id: { type: String },
348
+ geo_selection_id: { type: String },
349
+ geo_selection_title: { type: String },
350
+ type: { type: String },
351
+ title: { type: String },
352
+ data_short: { type: String },
353
+ data: { type: String },
354
+ created_at: { type: Number, default: Date.now },
355
+ updated_at: { type: Number, default: Date.now },
356
+ keywords: { type: [String], default: [] },
357
+ geo_selection: { type: mongoose_1.Schema.Types.Mixed },
358
+ raw_data: {
359
+ type: [
360
+ {
361
+ platform: { type: String, enum: types_1.platformsList },
362
+ source_id: { type: mongoose_1.Schema.Types.Mixed },
363
+ data_id: { type: mongoose_1.Schema.Types.Mixed },
364
+ data_url: { type: String },
365
+ data_text: { type: String },
366
+ author_username: { type: String },
367
+ author_id: { type: String },
368
+ timestamp: { type: Number },
369
+ data_geo: { type: [String], default: [] },
370
+ data_region_id: { type: String },
371
+ },
372
+ ],
373
+ default: undefined,
374
+ },
375
+ embeddings: { type: [Number], default: [] }, // vector embeddings for ML tasks
376
+ }, {
377
+ versionKey: false,
378
+ toJSON: { virtuals: true },
379
+ toObject: { virtuals: true },
380
+ });
package/dist/types.d.ts CHANGED
@@ -1,12 +1,12 @@
1
1
  import { z } from 'zod';
2
2
  export declare const platformsList: readonly ["telegram", "facebook", "instagram", "tiktok", "website", "youtube", "twitter", "x", "reddit", "linkedin", "snapchat", "whatsapp", "discord", "twitch"];
3
- export declare const entityTypesList: readonly ["profile", "group", "page", "channel", "hashtag", "website"];
3
+ export declare const entityTypesList: readonly ["profile", "group", "page", "channel", "hashtag", "website", "rss"];
4
4
  export declare const platformEntityMap: {
5
5
  readonly telegram: readonly ["channel", "group"];
6
6
  readonly facebook: readonly ["profile", "page", "group"];
7
7
  readonly instagram: readonly ["profile", "hashtag"];
8
8
  readonly tiktok: readonly ["profile", "hashtag"];
9
- readonly website: readonly ["website"];
9
+ readonly website: readonly ["website", "rss"];
10
10
  };
11
11
  export declare const priorityLevels: readonly ["none", "low", "medium", "high", "critical"];
12
12
  export declare const sourceStatusList: readonly ["active", "pending", "inactive", "requested", "discovered", "deleted"];
@@ -131,7 +131,7 @@ export declare const zodSourceSchema: z.ZodObject<{
131
131
  _id: z.ZodOptional<z.ZodString>;
132
132
  title: z.ZodOptional<z.ZodString>;
133
133
  platform: z.ZodEnum<["telegram", "facebook", "instagram", "tiktok", "website", "youtube", "twitter", "x", "reddit", "linkedin", "snapchat", "whatsapp", "discord", "twitch"]>;
134
- entity: z.ZodEnum<["profile", "group", "page", "channel", "hashtag", "website"]>;
134
+ entity: z.ZodEnum<["profile", "group", "page", "channel", "hashtag", "website", "rss"]>;
135
135
  public_id: z.ZodOptional<z.ZodUnion<[z.ZodString, z.ZodNumber]>>;
136
136
  url: z.ZodOptional<z.ZodString>;
137
137
  description: z.ZodOptional<z.ZodString>;
@@ -187,7 +187,7 @@ export declare const zodSourceSchema: z.ZodObject<{
187
187
  }, "strip", z.ZodTypeAny, {
188
188
  status: "active" | "pending" | "inactive" | "requested" | "discovered" | "deleted";
189
189
  platform: "telegram" | "facebook" | "instagram" | "tiktok" | "website" | "youtube" | "twitter" | "x" | "reddit" | "linkedin" | "snapchat" | "whatsapp" | "discord" | "twitch";
190
- entity: "website" | "profile" | "group" | "page" | "channel" | "hashtag";
190
+ entity: "website" | "profile" | "group" | "page" | "channel" | "hashtag" | "rss";
191
191
  is_public: boolean;
192
192
  errors: {
193
193
  is_handled: boolean;
@@ -224,7 +224,7 @@ export declare const zodSourceSchema: z.ZodObject<{
224
224
  }, {
225
225
  status: "active" | "pending" | "inactive" | "requested" | "discovered" | "deleted";
226
226
  platform: "telegram" | "facebook" | "instagram" | "tiktok" | "website" | "youtube" | "twitter" | "x" | "reddit" | "linkedin" | "snapchat" | "whatsapp" | "discord" | "twitch";
227
- entity: "website" | "profile" | "group" | "page" | "channel" | "hashtag";
227
+ entity: "website" | "profile" | "group" | "page" | "channel" | "hashtag" | "rss";
228
228
  errors: {
229
229
  message?: string | undefined;
230
230
  timestamp?: number | undefined;
@@ -2091,3 +2091,204 @@ export type FacebookMetadataType = z.infer<typeof facebookMetadataSchema>;
2091
2091
  export type InstagramMetadataType = z.infer<typeof instagramMetadataSchema>;
2092
2092
  export type TiktokMetadataType = z.infer<typeof tiktokMetadataSchema>;
2093
2093
  export type WebsiteMetadataType = z.infer<typeof websiteMetadataSchema>;
2094
+ /**
2095
+ * Harvester events
2096
+ */
2097
+ export declare const zodEventSchema: z.ZodObject<{
2098
+ _id: z.ZodOptional<z.ZodString>;
2099
+ region_id: z.ZodString;
2100
+ geo_selection_id: z.ZodOptional<z.ZodString>;
2101
+ geo_selection_title: z.ZodOptional<z.ZodString>;
2102
+ type: z.ZodString;
2103
+ title: z.ZodString;
2104
+ data_short: z.ZodOptional<z.ZodString>;
2105
+ data: z.ZodString;
2106
+ created_at: z.ZodNumber;
2107
+ updated_at: z.ZodNumber;
2108
+ keywords: z.ZodOptional<z.ZodArray<z.ZodString, "many">>;
2109
+ geo_selection: z.ZodObject<{
2110
+ _id: z.ZodOptional<z.ZodString>;
2111
+ title: z.ZodOptional<z.ZodString>;
2112
+ description: z.ZodOptional<z.ZodString>;
2113
+ type: z.ZodOptional<z.ZodString>;
2114
+ geos: z.ZodOptional<z.ZodArray<z.ZodString, "many">>;
2115
+ region_id: z.ZodOptional<z.ZodString>;
2116
+ notes: z.ZodOptional<z.ZodString>;
2117
+ status: z.ZodOptional<z.ZodEnum<["active", "inactive", "deleted"]>>;
2118
+ created_at: z.ZodOptional<z.ZodNumber>;
2119
+ updated_at: z.ZodOptional<z.ZodNumber>;
2120
+ coordinates: z.ZodOptional<z.ZodObject<{
2121
+ type: z.ZodOptional<z.ZodEnum<["point", "polygon"]>>;
2122
+ coordinates: z.ZodOptional<z.ZodUnion<[z.ZodTuple<[z.ZodNumber, z.ZodNumber], null>, z.ZodArray<z.ZodTuple<[z.ZodNumber, z.ZodNumber], null>, "many">]>>;
2123
+ }, "strip", z.ZodTypeAny, {
2124
+ type?: "point" | "polygon" | undefined;
2125
+ coordinates?: [number, number] | [number, number][] | undefined;
2126
+ }, {
2127
+ type?: "point" | "polygon" | undefined;
2128
+ coordinates?: [number, number] | [number, number][] | undefined;
2129
+ }>>;
2130
+ lat: z.ZodOptional<z.ZodNumber>;
2131
+ lng: z.ZodOptional<z.ZodNumber>;
2132
+ }, "strip", z.ZodTypeAny, {
2133
+ _id?: string | undefined;
2134
+ created_at?: number | undefined;
2135
+ updated_at?: number | undefined;
2136
+ status?: "active" | "inactive" | "deleted" | undefined;
2137
+ notes?: string | undefined;
2138
+ type?: string | undefined;
2139
+ coordinates?: {
2140
+ type?: "point" | "polygon" | undefined;
2141
+ coordinates?: [number, number] | [number, number][] | undefined;
2142
+ } | undefined;
2143
+ description?: string | undefined;
2144
+ region_id?: string | undefined;
2145
+ title?: string | undefined;
2146
+ geos?: string[] | undefined;
2147
+ lat?: number | undefined;
2148
+ lng?: number | undefined;
2149
+ }, {
2150
+ _id?: string | undefined;
2151
+ created_at?: number | undefined;
2152
+ updated_at?: number | undefined;
2153
+ status?: "active" | "inactive" | "deleted" | undefined;
2154
+ notes?: string | undefined;
2155
+ type?: string | undefined;
2156
+ coordinates?: {
2157
+ type?: "point" | "polygon" | undefined;
2158
+ coordinates?: [number, number] | [number, number][] | undefined;
2159
+ } | undefined;
2160
+ description?: string | undefined;
2161
+ region_id?: string | undefined;
2162
+ title?: string | undefined;
2163
+ geos?: string[] | undefined;
2164
+ lat?: number | undefined;
2165
+ lng?: number | undefined;
2166
+ }>;
2167
+ raw_data: z.ZodOptional<z.ZodArray<z.ZodObject<{
2168
+ _id: z.ZodOptional<z.ZodString>;
2169
+ platform: z.ZodEnum<["telegram", "facebook", "instagram", "tiktok", "website", "youtube", "twitter", "x", "reddit", "linkedin", "snapchat", "whatsapp", "discord", "twitch"]>;
2170
+ source_id: z.ZodUnion<[z.ZodString, z.ZodNumber]>;
2171
+ data_id: z.ZodUnion<[z.ZodString, z.ZodNumber]>;
2172
+ data_url: z.ZodOptional<z.ZodString>;
2173
+ data_text: z.ZodOptional<z.ZodString>;
2174
+ author_username: z.ZodOptional<z.ZodString>;
2175
+ author_id: z.ZodOptional<z.ZodString>;
2176
+ timestamp: z.ZodOptional<z.ZodNumber>;
2177
+ data_geo: z.ZodDefault<z.ZodOptional<z.ZodArray<z.ZodString, "many">>>;
2178
+ data_region_id: z.ZodOptional<z.ZodString>;
2179
+ }, "strip", z.ZodTypeAny, {
2180
+ platform: "telegram" | "facebook" | "instagram" | "tiktok" | "website" | "youtube" | "twitter" | "x" | "reddit" | "linkedin" | "snapchat" | "whatsapp" | "discord" | "twitch";
2181
+ source_id: string | number;
2182
+ data_id: string | number;
2183
+ data_geo: string[];
2184
+ _id?: string | undefined;
2185
+ timestamp?: number | undefined;
2186
+ data_url?: string | undefined;
2187
+ data_text?: string | undefined;
2188
+ author_username?: string | undefined;
2189
+ author_id?: string | undefined;
2190
+ data_region_id?: string | undefined;
2191
+ }, {
2192
+ platform: "telegram" | "facebook" | "instagram" | "tiktok" | "website" | "youtube" | "twitter" | "x" | "reddit" | "linkedin" | "snapchat" | "whatsapp" | "discord" | "twitch";
2193
+ source_id: string | number;
2194
+ data_id: string | number;
2195
+ _id?: string | undefined;
2196
+ timestamp?: number | undefined;
2197
+ data_url?: string | undefined;
2198
+ data_text?: string | undefined;
2199
+ data_geo?: string[] | undefined;
2200
+ author_username?: string | undefined;
2201
+ author_id?: string | undefined;
2202
+ data_region_id?: string | undefined;
2203
+ }>, "many">>;
2204
+ embeddings: z.ZodOptional<z.ZodArray<z.ZodNumber, "many">>;
2205
+ }, "strip", z.ZodTypeAny, {
2206
+ created_at: number;
2207
+ updated_at: number;
2208
+ type: string;
2209
+ region_id: string;
2210
+ title: string;
2211
+ data: string;
2212
+ geo_selection: {
2213
+ _id?: string | undefined;
2214
+ created_at?: number | undefined;
2215
+ updated_at?: number | undefined;
2216
+ status?: "active" | "inactive" | "deleted" | undefined;
2217
+ notes?: string | undefined;
2218
+ type?: string | undefined;
2219
+ coordinates?: {
2220
+ type?: "point" | "polygon" | undefined;
2221
+ coordinates?: [number, number] | [number, number][] | undefined;
2222
+ } | undefined;
2223
+ description?: string | undefined;
2224
+ region_id?: string | undefined;
2225
+ title?: string | undefined;
2226
+ geos?: string[] | undefined;
2227
+ lat?: number | undefined;
2228
+ lng?: number | undefined;
2229
+ };
2230
+ _id?: string | undefined;
2231
+ embeddings?: number[] | undefined;
2232
+ keywords?: string[] | undefined;
2233
+ geo_selection_id?: string | undefined;
2234
+ geo_selection_title?: string | undefined;
2235
+ data_short?: string | undefined;
2236
+ raw_data?: {
2237
+ platform: "telegram" | "facebook" | "instagram" | "tiktok" | "website" | "youtube" | "twitter" | "x" | "reddit" | "linkedin" | "snapchat" | "whatsapp" | "discord" | "twitch";
2238
+ source_id: string | number;
2239
+ data_id: string | number;
2240
+ data_geo: string[];
2241
+ _id?: string | undefined;
2242
+ timestamp?: number | undefined;
2243
+ data_url?: string | undefined;
2244
+ data_text?: string | undefined;
2245
+ author_username?: string | undefined;
2246
+ author_id?: string | undefined;
2247
+ data_region_id?: string | undefined;
2248
+ }[] | undefined;
2249
+ }, {
2250
+ created_at: number;
2251
+ updated_at: number;
2252
+ type: string;
2253
+ region_id: string;
2254
+ title: string;
2255
+ data: string;
2256
+ geo_selection: {
2257
+ _id?: string | undefined;
2258
+ created_at?: number | undefined;
2259
+ updated_at?: number | undefined;
2260
+ status?: "active" | "inactive" | "deleted" | undefined;
2261
+ notes?: string | undefined;
2262
+ type?: string | undefined;
2263
+ coordinates?: {
2264
+ type?: "point" | "polygon" | undefined;
2265
+ coordinates?: [number, number] | [number, number][] | undefined;
2266
+ } | undefined;
2267
+ description?: string | undefined;
2268
+ region_id?: string | undefined;
2269
+ title?: string | undefined;
2270
+ geos?: string[] | undefined;
2271
+ lat?: number | undefined;
2272
+ lng?: number | undefined;
2273
+ };
2274
+ _id?: string | undefined;
2275
+ embeddings?: number[] | undefined;
2276
+ keywords?: string[] | undefined;
2277
+ geo_selection_id?: string | undefined;
2278
+ geo_selection_title?: string | undefined;
2279
+ data_short?: string | undefined;
2280
+ raw_data?: {
2281
+ platform: "telegram" | "facebook" | "instagram" | "tiktok" | "website" | "youtube" | "twitter" | "x" | "reddit" | "linkedin" | "snapchat" | "whatsapp" | "discord" | "twitch";
2282
+ source_id: string | number;
2283
+ data_id: string | number;
2284
+ _id?: string | undefined;
2285
+ timestamp?: number | undefined;
2286
+ data_url?: string | undefined;
2287
+ data_text?: string | undefined;
2288
+ data_geo?: string[] | undefined;
2289
+ author_username?: string | undefined;
2290
+ author_id?: string | undefined;
2291
+ data_region_id?: string | undefined;
2292
+ }[] | undefined;
2293
+ }>;
2294
+ export type EventType = z.infer<typeof zodEventSchema>;
package/dist/types.js CHANGED
@@ -1,6 +1,6 @@
1
1
  "use strict";
2
2
  Object.defineProperty(exports, "__esModule", { value: true });
3
- exports.zodServerReportSchema = exports.zodRegionSummarySchema = exports.zodDashboardStatsSchema = exports.zodDataSchemaWithValidation = exports.zodDataSchema = exports.authorSchema = exports.repliesInfoSchema = exports.mediaItemSchema = exports.websiteMetadataSchema = exports.tiktokMetadataSchema = exports.instagramMetadataSchema = exports.facebookMetadataSchema = exports.telegramMetadataSchema = exports.zodGeoSelectionSchema = exports.zodGeoSchema = exports.zodSourceSchema = exports.zodSourceGroupSchema = exports.zodRegionSchema = exports.generalStatusList = exports.sourceStatusList = exports.priorityLevels = exports.platformEntityMap = exports.entityTypesList = exports.platformsList = void 0;
3
+ exports.zodEventSchema = exports.zodServerReportSchema = exports.zodRegionSummarySchema = exports.zodDashboardStatsSchema = exports.zodDataSchemaWithValidation = exports.zodDataSchema = exports.authorSchema = exports.repliesInfoSchema = exports.mediaItemSchema = exports.websiteMetadataSchema = exports.tiktokMetadataSchema = exports.instagramMetadataSchema = exports.facebookMetadataSchema = exports.telegramMetadataSchema = exports.zodGeoSelectionSchema = exports.zodGeoSchema = exports.zodSourceSchema = exports.zodSourceGroupSchema = exports.zodRegionSchema = exports.generalStatusList = exports.sourceStatusList = exports.priorityLevels = exports.platformEntityMap = exports.entityTypesList = exports.platformsList = void 0;
4
4
  const zod_1 = require("zod");
5
5
  exports.platformsList = [
6
6
  'telegram',
@@ -25,6 +25,7 @@ exports.entityTypesList = [
25
25
  'channel',
26
26
  'hashtag',
27
27
  'website',
28
+ 'rss',
28
29
  ];
29
30
  // Platform-specific entity mappings
30
31
  exports.platformEntityMap = {
@@ -32,7 +33,7 @@ exports.platformEntityMap = {
32
33
  facebook: ['profile', 'page', 'group'],
33
34
  instagram: ['profile', 'hashtag'],
34
35
  tiktok: ['profile', 'hashtag'],
35
- website: ['website'],
36
+ website: ['website', 'rss'],
36
37
  };
37
38
  exports.priorityLevels = [
38
39
  'none',
@@ -417,3 +418,37 @@ exports.zodServerReportSchema = zod_1.z.object({
417
418
  })
418
419
  .optional(),
419
420
  });
421
+ // -------------------
422
+ /**
423
+ * Harvester events
424
+ */
425
+ exports.zodEventSchema = zod_1.z.object({
426
+ _id: zod_1.z.string().optional(),
427
+ region_id: zod_1.z.string(),
428
+ geo_selection_id: zod_1.z.string().optional(),
429
+ geo_selection_title: zod_1.z.string().optional(),
430
+ type: zod_1.z.string(),
431
+ title: zod_1.z.string(),
432
+ data_short: zod_1.z.string().optional(),
433
+ data: zod_1.z.string(),
434
+ created_at: zod_1.z.number(),
435
+ updated_at: zod_1.z.number(),
436
+ keywords: zod_1.z.array(zod_1.z.string()).optional(),
437
+ geo_selection: exports.zodGeoSelectionSchema,
438
+ raw_data: zod_1.z
439
+ .array(zod_1.z.object({
440
+ _id: zod_1.z.string().optional(),
441
+ platform: zod_1.z.enum(exports.platformsList),
442
+ source_id: zod_1.z.string().or(zod_1.z.number()),
443
+ data_id: zod_1.z.string().or(zod_1.z.number()),
444
+ data_url: zod_1.z.string().url().optional(),
445
+ data_text: zod_1.z.string().optional(),
446
+ author_username: zod_1.z.string().optional(),
447
+ author_id: zod_1.z.string().optional(),
448
+ timestamp: zod_1.z.number().optional(),
449
+ data_geo: zod_1.z.array(zod_1.z.string()).optional().default([]),
450
+ data_region_id: zod_1.z.string().optional(),
451
+ }))
452
+ .optional(),
453
+ embeddings: zod_1.z.array(zod_1.z.number()).optional(), // e.g., vector embeddings for ML tasks
454
+ });
package/index.ts CHANGED
@@ -9,7 +9,6 @@ import { Schema } from 'mongoose';
9
9
  import {
10
10
  generalStatusList,
11
11
  platformsList,
12
- entityTypesList,
13
12
  sourceStatusList,
14
13
  priorityLevels,
15
14
  } from './types';
@@ -378,3 +377,44 @@ export const MongoServerReportSchema = new Schema(
378
377
  toObject: { virtuals: true },
379
378
  },
380
379
  );
380
+
381
+ export const MongoEventSchema = new Schema(
382
+ {
383
+ event_id: { type: String }, // unique identifier for the event
384
+ timestamp: { type: Number, default: Date.now }, // event creation timestamp
385
+ region_id: { type: String }, // e.g., region ID
386
+ geo_selection_id: { type: String }, // e.g., geo selection ID
387
+ geo_selection_title: { type: String }, // e.g., geo selection title
388
+ type: { type: String }, // e.g., 'alert', 'info'
389
+ title: { type: String }, // e.g., 'Red sea escalating'
390
+ data_short: { type: String }, // e.g., 'overview of the event'
391
+ data: { type: String }, // in depth data about the event, this string is updated each time the object is updated
392
+ created_at: { type: Number, default: Date.now }, // creation date
393
+ updated_at: { type: Number, default: Date.now }, // last update date
394
+ keywords: { type: [String], default: [] }, // e.g., ['red sea', 'escalating']
395
+ geo_selection: { type: Schema.Types.Mixed }, // embedded geo selection object
396
+ raw_data: {
397
+ type: [
398
+ {
399
+ platform: { type: String, enum: platformsList },
400
+ source_id: { type: Schema.Types.Mixed },
401
+ data_id: { type: Schema.Types.Mixed },
402
+ data_url: { type: String },
403
+ data_text: { type: String },
404
+ author_username: { type: String },
405
+ author_id: { type: String },
406
+ timestamp: { type: Number },
407
+ data_geo: { type: [String], default: [] },
408
+ data_region_id: { type: String },
409
+ },
410
+ ],
411
+ default: undefined,
412
+ }, // raw data items that triggered the event
413
+ embeddings: { type: [Number], default: [] }, // vector embeddings for ML tasks
414
+ },
415
+ {
416
+ versionKey: false,
417
+ toJSON: { virtuals: true },
418
+ toObject: { virtuals: true },
419
+ },
420
+ );
package/package.json CHANGED
@@ -1,6 +1,6 @@
1
1
  {
2
2
  "name": "harvester_sdk",
3
- "version": "1.0.53",
3
+ "version": "1.0.55",
4
4
  "description": "SDK for interacting with the Harvester API",
5
5
  "main": "dist/index.js",
6
6
  "types": "dist/index.d.ts",
package/types.ts CHANGED
@@ -24,6 +24,7 @@ export const entityTypesList = [
24
24
  'channel',
25
25
  'hashtag',
26
26
  'website',
27
+ 'rss',
27
28
  ] as const;
28
29
 
29
30
  // Platform-specific entity mappings
@@ -32,7 +33,7 @@ export const platformEntityMap = {
32
33
  facebook: ['profile', 'page', 'group'],
33
34
  instagram: ['profile', 'hashtag'],
34
35
  tiktok: ['profile', 'hashtag'],
35
- website: ['website'],
36
+ website: ['website', 'rss'],
36
37
  } as const;
37
38
 
38
39
  export const priorityLevels = [
@@ -96,7 +97,7 @@ export const zodSourceGroupSchema = z.object({
96
97
  error_count: z.number().optional(),
97
98
  total_posts: z.number().optional(),
98
99
  sources_discovered: z.number().optional(),
99
- })
100
+ }),
100
101
  )
101
102
  .optional(),
102
103
  });
@@ -481,3 +482,43 @@ export type FacebookMetadataType = z.infer<typeof facebookMetadataSchema>;
481
482
  export type InstagramMetadataType = z.infer<typeof instagramMetadataSchema>;
482
483
  export type TiktokMetadataType = z.infer<typeof tiktokMetadataSchema>;
483
484
  export type WebsiteMetadataType = z.infer<typeof websiteMetadataSchema>;
485
+
486
+ // -------------------
487
+ /**
488
+ * Harvester events
489
+ */
490
+
491
+ export const zodEventSchema = z.object({
492
+ _id: z.string().optional(),
493
+ region_id: z.string(), // e.g., region ID
494
+ geo_selection_id: z.string().optional(), // e.g., geo selection ID
495
+ geo_selection_title: z.string().optional(), // e.g., geo selection title
496
+ type: z.string(), // e.g., 'alert', 'info'
497
+ title: z.string(), // e.g., 'Red sea escalating'
498
+ data_short: z.string().optional(), // e.g., 'overview of the event'
499
+ data: z.string(), // in depth data about the event, this string is updated each time the object is updated
500
+ created_at: z.number(), // e.g., 1751210833000
501
+ updated_at: z.number(), // e.g., 1751210833000
502
+ keywords: z.array(z.string()).optional(), // e.g., ['red sea', 'escalating']
503
+ geo_selection: zodGeoSelectionSchema,
504
+ raw_data: z
505
+ .array(
506
+ z.object({
507
+ _id: z.string().optional(),
508
+ platform: z.enum(platformsList),
509
+ source_id: z.string().or(z.number()),
510
+ data_id: z.string().or(z.number()),
511
+ data_url: z.string().url().optional(),
512
+ data_text: z.string().optional(),
513
+ author_username: z.string().optional(),
514
+ author_id: z.string().optional(),
515
+ timestamp: z.number().optional(),
516
+ data_geo: z.array(z.string()).optional().default([]),
517
+ data_region_id: z.string().optional(),
518
+ }),
519
+ )
520
+ .optional(), // raw data items that triggered the event
521
+ embeddings: z.array(z.number()).optional(), // e.g., vector embeddings for ML tasks
522
+ });
523
+
524
+ export type EventType = z.infer<typeof zodEventSchema>;