harvester_sdk 1.0.7 → 1.0.8

package/dist/index.d.ts

@@ -36,6 +36,7 @@ export declare const MongoDataSchema: Schema<any, import("mongoose").Model<any,
 }, {
     created_at: number;
     updated_at: number;
+    source_dominant_geos: string[];
     source_id: any;
     data_geo: string[];
     media: import("mongoose").Types.DocumentArray<{
@@ -53,12 +54,12 @@ export declare const MongoDataSchema: Schema<any, import("mongoose").Model<any,
     }>;
     platform?: "telegram" | "facebook" | "instagram" | "tiktok" | "website" | null | undefined;
     metadata?: any;
-    group_id?: string | null | undefined;
     timestamp?: number | null | undefined;
     source_region_id?: string | null | undefined;
     source_region_title?: string | null | undefined;
     source_title?: string | null | undefined;
     source_url?: string | null | undefined;
+    source_group_id?: string | null | undefined;
     data_id?: any;
     data_text?: string | null | undefined;
     data_url?: string | null | undefined;
@@ -75,6 +76,7 @@ export declare const MongoDataSchema: Schema<any, import("mongoose").Model<any,
 }, import("mongoose").Document<unknown, {}, import("mongoose").FlatRecord<{
     created_at: number;
     updated_at: number;
+    source_dominant_geos: string[];
     source_id: any;
     data_geo: string[];
     media: import("mongoose").Types.DocumentArray<{
@@ -92,12 +94,12 @@ export declare const MongoDataSchema: Schema<any, import("mongoose").Model<any,
     }>;
     platform?: "telegram" | "facebook" | "instagram" | "tiktok" | "website" | null | undefined;
     metadata?: any;
-    group_id?: string | null | undefined;
     timestamp?: number | null | undefined;
     source_region_id?: string | null | undefined;
     source_region_title?: string | null | undefined;
     source_title?: string | null | undefined;
     source_url?: string | null | undefined;
+    source_group_id?: string | null | undefined;
     data_id?: any;
     data_text?: string | null | undefined;
     data_url?: string | null | undefined;
@@ -114,6 +116,7 @@ export declare const MongoDataSchema: Schema<any, import("mongoose").Model<any,
 }>, {}> & import("mongoose").FlatRecord<{
     created_at: number;
     updated_at: number;
+    source_dominant_geos: string[];
     source_id: any;
     data_geo: string[];
     media: import("mongoose").Types.DocumentArray<{
@@ -131,12 +134,12 @@ export declare const MongoDataSchema: Schema<any, import("mongoose").Model<any,
     }>;
     platform?: "telegram" | "facebook" | "instagram" | "tiktok" | "website" | null | undefined;
     metadata?: any;
-    group_id?: string | null | undefined;
     timestamp?: number | null | undefined;
     source_region_id?: string | null | undefined;
     source_region_title?: string | null | undefined;
     source_title?: string | null | undefined;
     source_url?: string | null | undefined;
+    source_group_id?: string | null | undefined;
     data_id?: any;
     data_text?: string | null | undefined;
     data_url?: string | null | undefined;
@@ -260,6 +263,7 @@ export declare const MongoSourceSchema: Schema<any, import("mongoose").Model<any
     public_id?: any;
     url?: string | null | undefined;
     language?: string | null | undefined;
+    region_title?: string | null | undefined;
     group_id?: string | null | undefined;
     requested_by?: {
         user_id?: string | null | undefined;
@@ -285,6 +289,7 @@ export declare const MongoSourceSchema: Schema<any, import("mongoose").Model<any
     public_id?: any;
     url?: string | null | undefined;
     language?: string | null | undefined;
+    region_title?: string | null | undefined;
     group_id?: string | null | undefined;
     requested_by?: {
         user_id?: string | null | undefined;
@@ -310,6 +315,7 @@ export declare const MongoSourceSchema: Schema<any, import("mongoose").Model<any
     public_id?: any;
     url?: string | null | undefined;
     language?: string | null | undefined;
+    region_title?: string | null | undefined;
     group_id?: string | null | undefined;
     requested_by?: {
         user_id?: string | null | undefined;

package/dist/index.js

@@ -27,9 +27,11 @@ exports.MongoDataSchema = new mongoose_1.Schema({
     },
     source_region_id: { type: String },
     source_region_title: { type: String },
+    source_dominant_geos: { type: [String], default: [] },
     source_id: { type: mongoose_1.Schema.Types.Mixed, required: true },
     source_title: { type: String },
     source_url: { type: String },
+    source_group_id: { type: String },
     data_id: { type: mongoose_1.Schema.Types.Mixed },
     data_geo: { type: [String], default: [] },
     data_text: { type: String },
@@ -55,8 +57,7 @@ exports.MongoDataSchema = new mongoose_1.Schema({
     author: { type: String },
     author_username: { type: String },
     replies: { type: mongoose_1.Schema.Types.Mixed },
-    author_id: { type: String },
-    group_id: { type: String }, // e.g., source group ID
+    author_id: { type: String }, // e.g., author ID
 }, {
     versionKey: false,
     toJSON: { virtuals: true },
@@ -129,6 +130,7 @@ exports.MongoSourceSchema = new mongoose_1.Schema({
     created_at: { type: Number, default: Date.now },
     updated_at: { type: Number, default: Date.now },
     region_id: { type: String },
+    region_title: { type: String },
     group_id: { type: String },
     dominant_geos: { type: [String], default: [] },
     requested_by: {

package/dist/types.d.ts

@@ -88,6 +88,7 @@ export declare const zodSourceSchema: z.ZodObject<{
     created_at: z.ZodOptional<z.ZodNumber>;
     updated_at: z.ZodOptional<z.ZodNumber>;
     region_id: z.ZodOptional<z.ZodString>;
+    region_title: z.ZodOptional<z.ZodString>;
     group_id: z.ZodOptional<z.ZodString>;
     dominant_geos: z.ZodOptional<z.ZodArray<z.ZodString, "many">>;
     requested_by: z.ZodOptional<z.ZodObject<{
@@ -127,6 +128,7 @@ export declare const zodSourceSchema: z.ZodObject<{
     language?: string | undefined;
     tags?: string[] | undefined;
     metadata?: Record<string, any> | undefined;
+    region_title?: string | undefined;
     group_id?: string | undefined;
     dominant_geos?: string[] | undefined;
     requested_by?: {
@@ -153,6 +155,7 @@ export declare const zodSourceSchema: z.ZodObject<{
     tags?: string[] | undefined;
     is_public?: boolean | undefined;
     metadata?: Record<string, any> | undefined;
+    region_title?: string | undefined;
     group_id?: string | undefined;
     dominant_geos?: string[] | undefined;
     requested_by?: {
@@ -222,9 +225,11 @@ export declare const zodDataSchema: z.ZodObject<{
     platform: z.ZodOptional<z.ZodEnum<["telegram", "facebook", "instagram", "tiktok", "website"]>>;
     source_region_id: z.ZodOptional<z.ZodString>;
     source_region_title: z.ZodOptional<z.ZodString>;
+    source_dominant_geos: z.ZodOptional<z.ZodArray<z.ZodString, "many">>;
     source_id: z.ZodUnion<[z.ZodString, z.ZodNumber]>;
     source_title: z.ZodOptional<z.ZodString>;
     source_url: z.ZodOptional<z.ZodString>;
+    source_group_id: z.ZodOptional<z.ZodString>;
     data_id: z.ZodOptional<z.ZodUnion<[z.ZodString, z.ZodNumber]>>;
     data_geo: z.ZodOptional<z.ZodArray<z.ZodString, "many">>;
     data_text: z.ZodOptional<z.ZodString>;
@@ -255,7 +260,6 @@ export declare const zodDataSchema: z.ZodObject<{
     author_username: z.ZodOptional<z.ZodString>;
     replies: z.ZodOptional<z.ZodAny>;
     author_id: z.ZodOptional<z.ZodString>;
-    group_id: z.ZodOptional<z.ZodString>;
 }, "strip", z.ZodTypeAny, {
     created_at: number;
     updated_at: number;
@@ -263,12 +267,13 @@ export declare const zodDataSchema: z.ZodObject<{
     _id?: string | undefined;
     platform?: "telegram" | "facebook" | "instagram" | "tiktok" | "website" | undefined;
     metadata?: Record<string, any> | undefined;
-    group_id?: string | undefined;
     timestamp?: number | undefined;
     source_region_id?: string | undefined;
     source_region_title?: string | undefined;
+    source_dominant_geos?: string[] | undefined;
     source_title?: string | undefined;
     source_url?: string | undefined;
+    source_group_id?: string | undefined;
     data_id?: string | number | undefined;
     data_geo?: string[] | undefined;
     data_text?: string | undefined;
@@ -295,12 +300,13 @@ export declare const zodDataSchema: z.ZodObject<{
     _id?: string | undefined;
     platform?: "telegram" | "facebook" | "instagram" | "tiktok" | "website" | undefined;
     metadata?: Record<string, any> | undefined;
-    group_id?: string | undefined;
     timestamp?: number | undefined;
     source_region_id?: string | undefined;
     source_region_title?: string | undefined;
+    source_dominant_geos?: string[] | undefined;
     source_title?: string | undefined;
     source_url?: string | undefined;
+    source_group_id?: string | undefined;
     data_id?: string | number | undefined;
     data_geo?: string[] | undefined;
     data_text?: string | undefined;

package/dist/types.js

@@ -72,6 +72,7 @@ exports.zodSourceSchema = zod_1.z.object({
     created_at: zod_1.z.number().optional(),
     updated_at: zod_1.z.number().optional(),
     region_id: zod_1.z.string().optional(),
+    region_title: zod_1.z.string().optional(),
     group_id: zod_1.z.string().optional(),
     dominant_geos: zod_1.z.array(zod_1.z.string()).optional(),
     requested_by: zod_1.z
@@ -110,9 +111,11 @@ exports.zodDataSchema = zod_1.z.object({
     platform: zod_1.z.enum(exports.platformsList).optional(),
     source_region_id: zod_1.z.string().optional(),
     source_region_title: zod_1.z.string().optional(),
+    source_dominant_geos: zod_1.z.array(zod_1.z.string()).optional(),
     source_id: zod_1.z.string().or(zod_1.z.number()),
     source_title: zod_1.z.string().optional(),
     source_url: zod_1.z.string().url().optional(),
+    source_group_id: zod_1.z.string().optional(),
     data_id: zod_1.z.string().or(zod_1.z.number()).optional(),
     data_geo: zod_1.z.array(zod_1.z.string()).optional(),
     data_text: zod_1.z.string().optional(),
@@ -136,8 +139,7 @@ exports.zodDataSchema = zod_1.z.object({
     author: zod_1.z.string().optional(),
     author_username: zod_1.z.string().optional(),
     replies: zod_1.z.any().optional(),
-    author_id: zod_1.z.string().optional(),
-    group_id: zod_1.z.string().optional(), // e.g., 'default_pipeline'
+    author_id: zod_1.z.string().optional(), // e.g., author ID -> get from post
     // translated_text: z.string().optional(), // translated text if available - most of the time it will be translated to English
     // entities: z.any().optional(), // array of reply texts or IDs
     // source_public_id: z.string().or(z.number()), // e.g., 'telegram:1234567890' (message_id)

package/index.ts

@@ -2,11 +2,11 @@
 export * from './types';
 
 import { Schema } from 'mongoose';
-import { 
-  generalStatusList, 
-  platformsList, 
+import {
+  generalStatusList,
+  platformsList,
   entityTypesList,
-  sourceStatusList 
+  sourceStatusList,
 } from './types';
 
 export const MongoDataSchema = new Schema(
@@ -18,9 +18,11 @@ export const MongoDataSchema = new Schema(
     }, // e.g., 'telegram', 'facebook'
     source_region_id: { type: String }, // e.g., 'hebron'
     source_region_title: { type: String }, // e.g., 'hebron'
+    source_dominant_geos: { type: [String], default: [] }, // e.g., ['hebron', 'west bank']
     source_id: { type: Schema.Types.Mixed, required: true }, // INDEX - reference to Source _id
     source_title: { type: String }, // e.g., 'Telegram Channel Name'
     source_url: { type: String }, // e.g., 'https://t.me/telegram_channel_name'
+    source_group_id: { type: String }, // e.g., source group ID
     data_id: { type: Schema.Types.Mixed }, // INDEX - original text ID (e.g., message_id)
     data_geo: { type: [String], default: [] }, // INDEX - e.g., ["sinjil", "ramallah", "west bank"]
     data_text: { type: String }, // processed text content
@@ -47,7 +49,6 @@ export const MongoDataSchema = new Schema(
     author_username: { type: String }, // e.g., author username
     replies: { type: Schema.Types.Mixed }, // array of reply texts or IDs
     author_id: { type: String }, // e.g., author ID
-    group_id: { type: String }, // e.g., source group ID
   },
   {
     versionKey: false,
@@ -69,7 +70,6 @@ export const MongoRegionSchema = new Schema(
       default: 'active',
     },
     notes: { type: String }, // internal notes about the region
-
   },
   {
     versionKey: false,
@@ -133,6 +133,7 @@ export const MongoSourceSchema = new Schema(
     created_at: { type: Number, default: Date.now },
     updated_at: { type: Number, default: Date.now },
     region_id: { type: String }, // e.g., region ID
+    region_title: { type: String }, // e.g., region title
     group_id: { type: String }, // e.g., source group ID
     dominant_geos: { type: [String], default: [] }, // e.g., dominant geos associated with this source
     requested_by: {

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "harvester_sdk",
-  "version": "1.0.7",
+  "version": "1.0.8",
   "description": "SDK for interacting with the Harvester API",
   "main": "dist/index.js",
   "types": "dist/index.d.ts",

package/types.ts

@@ -77,6 +77,7 @@ export const zodSourceSchema = z.object({
   created_at: z.number().optional(),
   updated_at: z.number().optional(),
   region_id: z.string().optional(), // e.g., region ID // INDEX
+  region_title: z.string().optional(), // e.g., region title
   group_id: z.string().optional(), // e.g., source group ID // INDEX
   dominant_geos: z.array(z.string()).optional(), // e.g., dominant geos associated with this source
   requested_by: z
@@ -116,23 +117,25 @@ export const zodDataSchema = z.object({
   _id: z.string().optional(),
   timestamp: z.number().optional(), // INDEX - date in milliseconds - e.g., 1751210833000
   platform: z.enum(platformsList).optional(), // e.g., 'telegram', 'facebook'
-  source_region_id: z.string().optional(), // e.g., 'hebron'
-  source_region_title: z.string().optional(), // e.g., 'hebron'
-  source_id: z.string().or(z.number()), // INDEX - reference to Source _id (e.g., '60c72b2f9b1e8d3f4c8b4567')
-  source_title: z.string().optional(), // e.g., 'Telegram Channel Name'
-  source_url: z.string().url().optional(), // e.g., 'https://t.me/telegram_channel_name'
-  data_id: z.string().or(z.number()).optional(), // INDEX - original text ID (e.g., message_id)
-  data_geo: z.array(z.string()).optional(), // INDEX - e.g., ["sinjil", "ramallah", "west bank"]
-  data_text: z.string().optional(), // processed text content
-  data_url: z.string().url().optional(), // original text URL if available
-  data_original_type: z.string().optional(), // e.g., 'post', 'comment', 'reply', 'video', 'image'
-  data_language: z.string().optional(), // detected language of the text
-  data_sentiment: z.string().optional(), // sentiment analysis result
-  data_timestamp: z.number().optional(), // original post timestamp if different from ingestion timestamp
-  is_reply: z.boolean().optional(), // true if this text is a reply to another text
-  reply_to_message_id: z.string().or(z.number()).optional(),
-  metadata: z.record(z.string(), z.any()).optional(), // platform-specific fields
-  created_at: z.number(),
+  source_region_id: z.string().optional(), // e.g., 'hebron' -> get from source object
+  source_region_title: z.string().optional(), // e.g., 'hebron' -> get from source object
+  source_dominant_geos: z.array(z.string()).optional(), // e.g., ['hebron', 'west bank'] -> get from source object
+  source_id: z.string().or(z.number()), // INDEX - reference to Source _id (e.g., '60c72b2f9b1e8d3f4c8b4567') -> get from source object
+  source_title: z.string().optional(), // e.g., 'Telegram Channel Name' -> get from source object
+  source_url: z.string().url().optional(), // e.g., 'https://t.me/telegram_channel_name' -> get from source object
+  source_group_id: z.string().optional(), // e.g., 'default_pipeline' -> get from source object
+  data_id: z.string().or(z.number()).optional(), // INDEX - original text ID (e.g., message_id) -> get from post
+  data_geo: z.array(z.string()).optional(), // INDEX - e.g., ["sinjil", "ramallah", "west bank"] -> get from processor
+  data_text: z.string().optional(), // processed text content -> get from post or processor
+  data_url: z.string().url().optional(), // original text URL if available -> get from post
+  data_original_type: z.string().optional(), // e.g., 'post', 'comment', 'reply', 'video', 'image' -> get from post
+  data_language: z.string().optional(), // detected language of the text -> get from post or processor
+  data_sentiment: z.string().optional(), // sentiment analysis result -> get from processor
+  data_timestamp: z.number().optional(), // original post timestamp if different from ingestion timestamp -> get from post
+  is_reply: z.boolean().optional(), // true if this text is a reply to another text -> get from post
+  reply_to_message_id: z.string().or(z.number()).optional(), // ID of the message this is a reply to -> get from post
+  metadata: z.record(z.string(), z.any()).optional(), // platform-specific fields -> get from post
+  created_at: z.number(), 
   updated_at: z.number(),
   media: z
     .array(
@@ -142,12 +145,11 @@ export const zodDataSchema = z.object({
         caption: z.string().optional(),
       })
     )
-    .optional(), // media attachments
-  author: z.string().optional(), // e.g., author name or ID
-  author_username: z.string().optional(), // e.g., author username
-  replies: z.any().optional(), // array of reply texts or IDs
-  author_id: z.string().optional(), // e.g., author ID
-  group_id: z.string().optional(), // e.g., 'default_pipeline'
+    .optional(), // media attachments -> get from post
+  author: z.string().optional(), // e.g., author name or ID -> get from post
+  author_username: z.string().optional(), // e.g., author username -> get from post
+  replies: z.any().optional(), // array of reply texts or IDs -> get from post
+  author_id: z.string().optional(), // e.g., author ID -> get from post
   // translated_text: z.string().optional(), // translated text if available - most of the time it will be translated to English
   // entities: z.any().optional(), // array of reply texts or IDs
   // source_public_id: z.string().or(z.number()), // e.g., 'telegram:1234567890' (message_id)