harvester_sdk 1.0.6 → 1.0.8

package/dist/index.d.ts

@@ -36,11 +36,9 @@ export declare const MongoDataSchema: Schema<any, import("mongoose").Model<any,
 }, {
     created_at: number;
     updated_at: number;
-    platform: "telegram" | "facebook" | "instagram" | "tiktok" | "website";
-    timestamp: number;
-    text_geo: string[];
+    source_dominant_geos: string[];
     source_id: any;
-    source_name: string;
+    data_geo: string[];
     media: import("mongoose").Types.DocumentArray<{
         type?: "image" | "video" | "audio" | "link" | null | undefined;
         url?: string | null | undefined;
@@ -54,31 +52,33 @@ export declare const MongoDataSchema: Schema<any, import("mongoose").Model<any,
         url?: string | null | undefined;
         caption?: string | null | undefined;
     }>;
-    language?: string | null | undefined;
+    platform?: "telegram" | "facebook" | "instagram" | "tiktok" | "website" | null | undefined;
     metadata?: any;
-    source_region?: string | null | undefined;
-    source_public_id?: any;
-    platform_id?: any;
-    original_text_id?: any;
-    original_text?: string | null | undefined;
-    translated_text?: string | null | undefined;
+    timestamp?: number | null | undefined;
+    source_region_id?: string | null | undefined;
+    source_region_title?: string | null | undefined;
+    source_title?: string | null | undefined;
+    source_url?: string | null | undefined;
+    source_group_id?: string | null | undefined;
+    data_id?: any;
+    data_text?: string | null | undefined;
+    data_url?: string | null | undefined;
+    data_original_type?: string | null | undefined;
+    data_language?: string | null | undefined;
+    data_sentiment?: string | null | undefined;
+    data_timestamp?: number | null | undefined;
     is_reply?: boolean | null | undefined;
     reply_to_message_id?: any;
     author?: string | null | undefined;
-    replies?: any;
-    entities?: any;
     author_username?: string | null | undefined;
+    replies?: any;
     author_id?: string | null | undefined;
-    source_geo?: string | null | undefined;
-    pipeline_name?: string | null | undefined;
 }, import("mongoose").Document<unknown, {}, import("mongoose").FlatRecord<{
     created_at: number;
     updated_at: number;
-    platform: "telegram" | "facebook" | "instagram" | "tiktok" | "website";
-    timestamp: number;
-    text_geo: string[];
+    source_dominant_geos: string[];
     source_id: any;
-    source_name: string;
+    data_geo: string[];
     media: import("mongoose").Types.DocumentArray<{
         type?: "image" | "video" | "audio" | "link" | null | undefined;
         url?: string | null | undefined;
@@ -92,31 +92,33 @@ export declare const MongoDataSchema: Schema<any, import("mongoose").Model<any,
         url?: string | null | undefined;
         caption?: string | null | undefined;
     }>;
-    language?: string | null | undefined;
+    platform?: "telegram" | "facebook" | "instagram" | "tiktok" | "website" | null | undefined;
     metadata?: any;
-    source_region?: string | null | undefined;
-    source_public_id?: any;
-    platform_id?: any;
-    original_text_id?: any;
-    original_text?: string | null | undefined;
-    translated_text?: string | null | undefined;
+    timestamp?: number | null | undefined;
+    source_region_id?: string | null | undefined;
+    source_region_title?: string | null | undefined;
+    source_title?: string | null | undefined;
+    source_url?: string | null | undefined;
+    source_group_id?: string | null | undefined;
+    data_id?: any;
+    data_text?: string | null | undefined;
+    data_url?: string | null | undefined;
+    data_original_type?: string | null | undefined;
+    data_language?: string | null | undefined;
+    data_sentiment?: string | null | undefined;
+    data_timestamp?: number | null | undefined;
     is_reply?: boolean | null | undefined;
     reply_to_message_id?: any;
     author?: string | null | undefined;
-    replies?: any;
-    entities?: any;
     author_username?: string | null | undefined;
+    replies?: any;
     author_id?: string | null | undefined;
-    source_geo?: string | null | undefined;
-    pipeline_name?: string | null | undefined;
 }>, {}> & import("mongoose").FlatRecord<{
     created_at: number;
     updated_at: number;
-    platform: "telegram" | "facebook" | "instagram" | "tiktok" | "website";
-    timestamp: number;
-    text_geo: string[];
+    source_dominant_geos: string[];
     source_id: any;
-    source_name: string;
+    data_geo: string[];
     media: import("mongoose").Types.DocumentArray<{
         type?: "image" | "video" | "audio" | "link" | null | undefined;
         url?: string | null | undefined;
@@ -130,23 +132,27 @@ export declare const MongoDataSchema: Schema<any, import("mongoose").Model<any,
         url?: string | null | undefined;
         caption?: string | null | undefined;
     }>;
-    language?: string | null | undefined;
+    platform?: "telegram" | "facebook" | "instagram" | "tiktok" | "website" | null | undefined;
     metadata?: any;
-    source_region?: string | null | undefined;
-    source_public_id?: any;
-    platform_id?: any;
-    original_text_id?: any;
-    original_text?: string | null | undefined;
-    translated_text?: string | null | undefined;
+    timestamp?: number | null | undefined;
+    source_region_id?: string | null | undefined;
+    source_region_title?: string | null | undefined;
+    source_title?: string | null | undefined;
+    source_url?: string | null | undefined;
+    source_group_id?: string | null | undefined;
+    data_id?: any;
+    data_text?: string | null | undefined;
+    data_url?: string | null | undefined;
+    data_original_type?: string | null | undefined;
+    data_language?: string | null | undefined;
+    data_sentiment?: string | null | undefined;
+    data_timestamp?: number | null | undefined;
     is_reply?: boolean | null | undefined;
     reply_to_message_id?: any;
     author?: string | null | undefined;
-    replies?: any;
-    entities?: any;
     author_username?: string | null | undefined;
+    replies?: any;
     author_id?: string | null | undefined;
-    source_geo?: string | null | undefined;
-    pipeline_name?: string | null | undefined;
 }> & {
     _id: import("mongoose").Types.ObjectId;
 } & {
@@ -257,6 +263,7 @@ export declare const MongoSourceSchema: Schema<any, import("mongoose").Model<any
     public_id?: any;
     url?: string | null | undefined;
     language?: string | null | undefined;
+    region_title?: string | null | undefined;
     group_id?: string | null | undefined;
     requested_by?: {
         user_id?: string | null | undefined;
@@ -282,6 +289,7 @@ export declare const MongoSourceSchema: Schema<any, import("mongoose").Model<any
     public_id?: any;
     url?: string | null | undefined;
     language?: string | null | undefined;
+    region_title?: string | null | undefined;
     group_id?: string | null | undefined;
     requested_by?: {
         user_id?: string | null | undefined;
@@ -307,6 +315,7 @@ export declare const MongoSourceSchema: Schema<any, import("mongoose").Model<any
     public_id?: any;
     url?: string | null | undefined;
     language?: string | null | undefined;
+    region_title?: string | null | undefined;
     group_id?: string | null | undefined;
     requested_by?: {
         user_id?: string | null | undefined;
@@ -320,193 +329,6 @@ export declare const MongoSourceSchema: Schema<any, import("mongoose").Model<any
 } & {
     __v: number;
 }>;
-export declare const MongoQuerySchema: Schema<any, import("mongoose").Model<any, any, any, any, any, any>, {}, {}, {}, {}, {
-    versionKey: false;
-    toJSON: {
-        virtuals: true;
-    };
-    toObject: {
-        virtuals: true;
-    };
-}, {
-    user_id: string;
-    timestamp: number;
-    geos: string[];
-    sources: string[];
-    geos_ids: string[];
-    time_range: any;
-    title?: string | null | undefined;
-    query?: string | null | undefined;
-    user_instructions?: string | null | undefined;
-    user_time_zone?: string | null | undefined;
-}, import("mongoose").Document<unknown, {}, import("mongoose").FlatRecord<{
-    user_id: string;
-    timestamp: number;
-    geos: string[];
-    sources: string[];
-    geos_ids: string[];
-    time_range: any;
-    title?: string | null | undefined;
-    query?: string | null | undefined;
-    user_instructions?: string | null | undefined;
-    user_time_zone?: string | null | undefined;
-}>, {}> & import("mongoose").FlatRecord<{
-    user_id: string;
-    timestamp: number;
-    geos: string[];
-    sources: string[];
-    geos_ids: string[];
-    time_range: any;
-    title?: string | null | undefined;
-    query?: string | null | undefined;
-    user_instructions?: string | null | undefined;
-    user_time_zone?: string | null | undefined;
-}> & {
-    _id: import("mongoose").Types.ObjectId;
-} & {
-    __v: number;
-}>;
-export declare const MongoUserSettingsSchema: Schema<any, import("mongoose").Model<any, any, any, any, any, any>, {}, {}, {}, {}, {
-    versionKey: false;
-    toJSON: {
-        virtuals: true;
-    };
-    toObject: {
-        virtuals: true;
-    };
-}, {
-    created_at: NativeDate;
-    updated_at: NativeDate;
-    status: "active" | "inactive" | "deleted";
-    user_id: string;
-    geos: string[];
-    sources: string[];
-    has_jobs_access: boolean;
-    active_jobs_limit: number;
-    time_range?: any;
-    thread_id?: string | null | undefined;
-    instructions?: string | null | undefined;
-}, import("mongoose").Document<unknown, {}, import("mongoose").FlatRecord<{
-    created_at: NativeDate;
-    updated_at: NativeDate;
-    status: "active" | "inactive" | "deleted";
-    user_id: string;
-    geos: string[];
-    sources: string[];
-    has_jobs_access: boolean;
-    active_jobs_limit: number;
-    time_range?: any;
-    thread_id?: string | null | undefined;
-    instructions?: string | null | undefined;
-}>, {}> & import("mongoose").FlatRecord<{
-    created_at: NativeDate;
-    updated_at: NativeDate;
-    status: "active" | "inactive" | "deleted";
-    user_id: string;
-    geos: string[];
-    sources: string[];
-    has_jobs_access: boolean;
-    active_jobs_limit: number;
-    time_range?: any;
-    thread_id?: string | null | undefined;
-    instructions?: string | null | undefined;
-}> & {
-    _id: import("mongoose").Types.ObjectId;
-} & {
-    __v: number;
-}>;
-export declare const MongoConversationSchema: Schema<any, import("mongoose").Model<any, any, any, any, any, any>, {}, {}, {}, {}, {
-    versionKey: false;
-    toJSON: {
-        virtuals: true;
-    };
-    toObject: {
-        virtuals: true;
-    };
-}, {
-    created_at: NativeDate;
-    updated_at: NativeDate;
-    status: "active" | "inactive" | "deleted";
-    user_id: string;
-    is_job: boolean;
-    title?: string | null | undefined;
-    thread_id?: string | null | undefined;
-}, import("mongoose").Document<unknown, {}, import("mongoose").FlatRecord<{
-    created_at: NativeDate;
-    updated_at: NativeDate;
-    status: "active" | "inactive" | "deleted";
-    user_id: string;
-    is_job: boolean;
-    title?: string | null | undefined;
-    thread_id?: string | null | undefined;
-}>, {}> & import("mongoose").FlatRecord<{
-    created_at: NativeDate;
-    updated_at: NativeDate;
-    status: "active" | "inactive" | "deleted";
-    user_id: string;
-    is_job: boolean;
-    title?: string | null | undefined;
-    thread_id?: string | null | undefined;
-}> & {
-    _id: import("mongoose").Types.ObjectId;
-} & {
-    __v: number;
-}>;
-export declare const MongoJobSchema: Schema<any, import("mongoose").Model<any, any, any, any, any, any>, {}, {}, {}, {}, {
-    versionKey: false;
-    toJSON: {
-        virtuals: true;
-    };
-    toObject: {
-        virtuals: true;
-    };
-}, {
-    created_at: number;
-    updated_at: number;
-    status: "active" | "inactive" | "deleted";
-    user_id: string;
-    query: any;
-    job_name: string;
-    thread_id?: string | null | undefined;
-    job_description?: string | null | undefined;
-    schedule?: string | null | undefined;
-    schedule_text?: string | null | undefined;
-    active_until?: number | null | undefined;
-    conversation_id?: string | null | undefined;
-    time_zone?: string | null | undefined;
-}, import("mongoose").Document<unknown, {}, import("mongoose").FlatRecord<{
-    created_at: number;
-    updated_at: number;
-    status: "active" | "inactive" | "deleted";
-    user_id: string;
-    query: any;
-    job_name: string;
-    thread_id?: string | null | undefined;
-    job_description?: string | null | undefined;
-    schedule?: string | null | undefined;
-    schedule_text?: string | null | undefined;
-    active_until?: number | null | undefined;
-    conversation_id?: string | null | undefined;
-    time_zone?: string | null | undefined;
-}>, {}> & import("mongoose").FlatRecord<{
-    created_at: number;
-    updated_at: number;
-    status: "active" | "inactive" | "deleted";
-    user_id: string;
-    query: any;
-    job_name: string;
-    thread_id?: string | null | undefined;
-    job_description?: string | null | undefined;
-    schedule?: string | null | undefined;
-    schedule_text?: string | null | undefined;
-    active_until?: number | null | undefined;
-    conversation_id?: string | null | undefined;
-    time_zone?: string | null | undefined;
-}> & {
-    _id: import("mongoose").Types.ObjectId;
-} & {
-    __v: number;
-}>;
 export declare const MongoGeoSchema: Schema<any, import("mongoose").Model<any, any, any, any, any, any>, {}, {}, {}, {}, {
     versionKey: false;
     toJSON: {

package/dist/index.js

@@ -14,33 +14,37 @@ var __exportStar = (this && this.__exportStar) || function(m, exports) {
     for (var p in m) if (p !== "default" && !Object.prototype.hasOwnProperty.call(exports, p)) __createBinding(exports, m, p);
 };
 Object.defineProperty(exports, "__esModule", { value: true });
-exports.MongoApiKeySchema = exports.MongoGeoSelectionSchema = exports.MongoGeoSchema = exports.MongoJobSchema = exports.MongoConversationSchema = exports.MongoUserSettingsSchema = exports.MongoQuerySchema = exports.MongoSourceSchema = exports.MongoSourceGroupSchema = exports.MongoRegionSchema = exports.MongoDataSchema = void 0;
+exports.MongoApiKeySchema = exports.MongoGeoSelectionSchema = exports.MongoGeoSchema = exports.MongoSourceSchema = exports.MongoSourceGroupSchema = exports.MongoRegionSchema = exports.MongoDataSchema = void 0;
 // Export types for consumers
 __exportStar(require("./types"), exports);
 const mongoose_1 = require("mongoose");
 const types_1 = require("./types");
 exports.MongoDataSchema = new mongoose_1.Schema({
-    source_id: { type: mongoose_1.Schema.Types.Mixed, required: true },
-    source_public_id: { type: mongoose_1.Schema.Types.Mixed },
-    source_name: { type: String, required: true },
-    platform_id: { type: mongoose_1.Schema.Types.Mixed },
+    timestamp: { type: Number },
     platform: {
         type: String,
         enum: types_1.platformsList,
-        required: true,
     },
-    original_text_id: { type: mongoose_1.Schema.Types.Mixed },
-    original_text: { type: String },
-    translated_text: { type: String },
-    timestamp: { type: Number, required: true },
-    language: { type: String },
-    text_geo: { type: [String], default: [] },
-    source_geo: { type: String },
+    source_region_id: { type: String },
+    source_region_title: { type: String },
+    source_dominant_geos: { type: [String], default: [] },
+    source_id: { type: mongoose_1.Schema.Types.Mixed, required: true },
+    source_title: { type: String },
+    source_url: { type: String },
+    source_group_id: { type: String },
+    data_id: { type: mongoose_1.Schema.Types.Mixed },
+    data_geo: { type: [String], default: [] },
+    data_text: { type: String },
+    data_url: { type: String },
+    data_original_type: { type: String },
+    data_language: { type: String },
+    data_sentiment: { type: String },
+    data_timestamp: { type: Number },
     is_reply: { type: Boolean },
     reply_to_message_id: { type: mongoose_1.Schema.Types.Mixed },
     metadata: { type: Object },
-    created_at: { type: Number, default: Date.now },
-    updated_at: { type: Number, default: Date.now },
+    created_at: { type: Number, default: Date.now, required: true },
+    updated_at: { type: Number, default: Date.now, required: true },
     media: {
         type: [
             {
@@ -51,12 +55,9 @@ exports.MongoDataSchema = new mongoose_1.Schema({
         ],
     },
     author: { type: String },
-    replies: { type: mongoose_1.Schema.Types.Mixed },
-    entities: { type: mongoose_1.Schema.Types.Mixed },
     author_username: { type: String },
-    author_id: { type: String },
-    pipeline_name: { type: String },
-    source_region: { type: String }, // e.g., "New York", "California"
+    replies: { type: mongoose_1.Schema.Types.Mixed },
+    author_id: { type: String }, // e.g., author ID
 }, {
     versionKey: false,
     toJSON: { virtuals: true },
@@ -129,6 +130,7 @@ exports.MongoSourceSchema = new mongoose_1.Schema({
     created_at: { type: Number, default: Date.now },
     updated_at: { type: Number, default: Date.now },
     region_id: { type: String },
+    region_title: { type: String },
     group_id: { type: String },
     dominant_geos: { type: [String], default: [] },
     requested_by: {
@@ -147,88 +149,6 @@ exports.MongoSourceSchema = new mongoose_1.Schema({
     toJSON: { virtuals: true },
     toObject: { virtuals: true },
 });
-exports.MongoQuerySchema = new mongoose_1.Schema({
-    title: { type: String },
-    query: { type: String },
-    sources: { type: [String] },
-    geos: { type: [String], required: true },
-    geos_ids: { type: [String], required: true },
-    time_range: {
-        type: mongoose_1.Schema.Types.Mixed,
-        required: true,
-    },
-    user_id: { type: String, required: true },
-    user_instructions: { type: String },
-    user_time_zone: { type: String },
-    timestamp: { type: Number, default: Date.now }, // creation date
-}, {
-    versionKey: false,
-    toJSON: { virtuals: true },
-    toObject: { virtuals: true },
-});
-exports.MongoUserSettingsSchema = new mongoose_1.Schema({
-    sources: { type: [String] },
-    geos: { type: [String], default: [] },
-    time_range: {
-        type: mongoose_1.Schema.Types.Mixed,
-    },
-    user_id: { type: String, required: true },
-    thread_id: { type: String },
-    instructions: { type: String },
-    has_jobs_access: { type: Boolean, default: false },
-    active_jobs_limit: { type: Number, default: 5 },
-    created_at: { type: Date, default: Date.now },
-    updated_at: { type: Date, default: Date.now },
-    status: {
-        type: String,
-        enum: types_1.generalStatusList,
-        default: 'active',
-    }, // status of the user
-}, {
-    versionKey: false,
-    toJSON: { virtuals: true },
-    toObject: { virtuals: true },
-});
-exports.MongoConversationSchema = new mongoose_1.Schema({
-    user_id: { type: String, required: true },
-    title: { type: String },
-    thread_id: { type: String },
-    status: {
-        type: String,
-        enum: types_1.generalStatusList,
-        default: 'active',
-    },
-    created_at: { type: Date, default: Date.now },
-    updated_at: { type: Date, default: Date.now },
-    is_job: { type: Boolean, default: false }, // true if this is a job conversation
-}, {
-    versionKey: false,
-    toJSON: { virtuals: true },
-    toObject: { virtuals: true },
-});
-exports.MongoJobSchema = new mongoose_1.Schema({
-    user_id: { type: String, required: true },
-    job_name: { type: String, required: true },
-    job_description: { type: String },
-    status: {
-        type: String,
-        enum: types_1.generalStatusList,
-        default: 'active',
-    },
-    created_at: { type: Number, default: Date.now },
-    updated_at: { type: Number, default: Date.now },
-    schedule: { type: String },
-    schedule_text: { type: String },
-    active_until: { type: Number || undefined },
-    query: { type: mongoose_1.Schema.Types.Mixed, required: true },
-    conversation_id: { type: String },
-    thread_id: { type: String },
-    time_zone: { type: String }, // optional timezone for the job
-}, {
-    versionKey: false,
-    toJSON: { virtuals: true },
-    toObject: { virtuals: true },
-});
 exports.MongoGeoSchema = new mongoose_1.Schema({
     geo_text: { type: String },
     timestamp: { type: Number },

package/dist/types.d.ts

@@ -88,6 +88,7 @@ export declare const zodSourceSchema: z.ZodObject<{
     created_at: z.ZodOptional<z.ZodNumber>;
     updated_at: z.ZodOptional<z.ZodNumber>;
     region_id: z.ZodOptional<z.ZodString>;
+    region_title: z.ZodOptional<z.ZodString>;
     group_id: z.ZodOptional<z.ZodString>;
     dominant_geos: z.ZodOptional<z.ZodArray<z.ZodString, "many">>;
     requested_by: z.ZodOptional<z.ZodObject<{
@@ -127,6 +128,7 @@ export declare const zodSourceSchema: z.ZodObject<{
     language?: string | undefined;
     tags?: string[] | undefined;
     metadata?: Record<string, any> | undefined;
+    region_title?: string | undefined;
     group_id?: string | undefined;
     dominant_geos?: string[] | undefined;
     requested_by?: {
@@ -153,6 +155,7 @@ export declare const zodSourceSchema: z.ZodObject<{
     tags?: string[] | undefined;
     is_public?: boolean | undefined;
     metadata?: Record<string, any> | undefined;
+    region_title?: string | undefined;
     group_id?: string | undefined;
     dominant_geos?: string[] | undefined;
     requested_by?: {
@@ -217,19 +220,24 @@ export declare const zodGeoSelectionSchema: z.ZodObject<{
     geos?: string[] | undefined;
 }>;
 export declare const zodDataSchema: z.ZodObject<{
-    text_geo: z.ZodArray<z.ZodString, "many">;
-    timestamp: z.ZodNumber;
-    platform: z.ZodEnum<["telegram", "facebook", "instagram", "tiktok", "website"]>;
-    source_region: z.ZodOptional<z.ZodString>;
     _id: z.ZodOptional<z.ZodString>;
+    timestamp: z.ZodOptional<z.ZodNumber>;
+    platform: z.ZodOptional<z.ZodEnum<["telegram", "facebook", "instagram", "tiktok", "website"]>>;
+    source_region_id: z.ZodOptional<z.ZodString>;
+    source_region_title: z.ZodOptional<z.ZodString>;
+    source_dominant_geos: z.ZodOptional<z.ZodArray<z.ZodString, "many">>;
     source_id: z.ZodUnion<[z.ZodString, z.ZodNumber]>;
-    source_public_id: z.ZodUnion<[z.ZodString, z.ZodNumber]>;
-    source_name: z.ZodString;
-    platform_id: z.ZodUnion<[z.ZodString, z.ZodNumber]>;
-    original_text_id: z.ZodUnion<[z.ZodString, z.ZodNumber]>;
-    original_text: z.ZodOptional<z.ZodString>;
-    translated_text: z.ZodOptional<z.ZodString>;
-    language: z.ZodOptional<z.ZodString>;
+    source_title: z.ZodOptional<z.ZodString>;
+    source_url: z.ZodOptional<z.ZodString>;
+    source_group_id: z.ZodOptional<z.ZodString>;
+    data_id: z.ZodOptional<z.ZodUnion<[z.ZodString, z.ZodNumber]>>;
+    data_geo: z.ZodOptional<z.ZodArray<z.ZodString, "many">>;
+    data_text: z.ZodOptional<z.ZodString>;
+    data_url: z.ZodOptional<z.ZodString>;
+    data_original_type: z.ZodOptional<z.ZodString>;
+    data_language: z.ZodOptional<z.ZodString>;
+    data_sentiment: z.ZodOptional<z.ZodString>;
+    data_timestamp: z.ZodOptional<z.ZodNumber>;
     is_reply: z.ZodOptional<z.ZodBoolean>;
     reply_to_message_id: z.ZodOptional<z.ZodUnion<[z.ZodString, z.ZodNumber]>>;
     metadata: z.ZodOptional<z.ZodRecord<z.ZodString, z.ZodAny>>;
@@ -249,29 +257,31 @@ export declare const zodDataSchema: z.ZodObject<{
         caption?: string | undefined;
     }>, "many">>;
     author: z.ZodOptional<z.ZodString>;
-    replies: z.ZodOptional<z.ZodAny>;
-    entities: z.ZodOptional<z.ZodAny>;
     author_username: z.ZodOptional<z.ZodString>;
+    replies: z.ZodOptional<z.ZodAny>;
     author_id: z.ZodOptional<z.ZodString>;
-    group_id: z.ZodOptional<z.ZodString>;
 }, "strip", z.ZodTypeAny, {
     created_at: number;
     updated_at: number;
-    platform: "telegram" | "facebook" | "instagram" | "tiktok" | "website";
-    timestamp: number;
-    text_geo: string[];
     source_id: string | number;
-    source_public_id: string | number;
-    source_name: string;
-    platform_id: string | number;
-    original_text_id: string | number;
     _id?: string | undefined;
-    language?: string | undefined;
+    platform?: "telegram" | "facebook" | "instagram" | "tiktok" | "website" | undefined;
     metadata?: Record<string, any> | undefined;
-    group_id?: string | undefined;
-    source_region?: string | undefined;
-    original_text?: string | undefined;
-    translated_text?: string | undefined;
+    timestamp?: number | undefined;
+    source_region_id?: string | undefined;
+    source_region_title?: string | undefined;
+    source_dominant_geos?: string[] | undefined;
+    source_title?: string | undefined;
+    source_url?: string | undefined;
+    source_group_id?: string | undefined;
+    data_id?: string | number | undefined;
+    data_geo?: string[] | undefined;
+    data_text?: string | undefined;
+    data_url?: string | undefined;
+    data_original_type?: string | undefined;
+    data_language?: string | undefined;
+    data_sentiment?: string | undefined;
+    data_timestamp?: number | undefined;
     is_reply?: boolean | undefined;
     reply_to_message_id?: string | number | undefined;
     media?: {
@@ -280,28 +290,31 @@ export declare const zodDataSchema: z.ZodObject<{
         caption?: string | undefined;
     }[] | undefined;
     author?: string | undefined;
-    replies?: any;
-    entities?: any;
     author_username?: string | undefined;
+    replies?: any;
     author_id?: string | undefined;
 }, {
     created_at: number;
     updated_at: number;
-    platform: "telegram" | "facebook" | "instagram" | "tiktok" | "website";
-    timestamp: number;
-    text_geo: string[];
     source_id: string | number;
-    source_public_id: string | number;
-    source_name: string;
-    platform_id: string | number;
-    original_text_id: string | number;
     _id?: string | undefined;
-    language?: string | undefined;
+    platform?: "telegram" | "facebook" | "instagram" | "tiktok" | "website" | undefined;
     metadata?: Record<string, any> | undefined;
-    group_id?: string | undefined;
-    source_region?: string | undefined;
-    original_text?: string | undefined;
-    translated_text?: string | undefined;
+    timestamp?: number | undefined;
+    source_region_id?: string | undefined;
+    source_region_title?: string | undefined;
+    source_dominant_geos?: string[] | undefined;
+    source_title?: string | undefined;
+    source_url?: string | undefined;
+    source_group_id?: string | undefined;
+    data_id?: string | number | undefined;
+    data_geo?: string[] | undefined;
+    data_text?: string | undefined;
+    data_url?: string | undefined;
+    data_original_type?: string | undefined;
+    data_language?: string | undefined;
+    data_sentiment?: string | undefined;
+    data_timestamp?: number | undefined;
     is_reply?: boolean | undefined;
     reply_to_message_id?: string | number | undefined;
     media?: {
@@ -310,18 +323,10 @@ export declare const zodDataSchema: z.ZodObject<{
         caption?: string | undefined;
     }[] | undefined;
     author?: string | undefined;
-    replies?: any;
-    entities?: any;
     author_username?: string | undefined;
+    replies?: any;
     author_id?: string | undefined;
 }>;
-/**
- * 'approved' - active and approved sources,
- * 'back_to_business' - sources that were paused and now resumed,
- * 'pending' - sources that are pending approval,
- * 'inactive' - sources that are inactive,
- * 'in_review' - sources that are under review
- */
 export type RegionType = z.infer<typeof zodRegionSchema>;
 export type SourceGroupType = z.infer<typeof zodSourceGroupSchema>;
 export type SourceType = z.infer<typeof zodSourceSchema>;

package/dist/types.js

@@ -72,6 +72,7 @@ exports.zodSourceSchema = zod_1.z.object({
     created_at: zod_1.z.number().optional(),
     updated_at: zod_1.z.number().optional(),
     region_id: zod_1.z.string().optional(),
+    region_title: zod_1.z.string().optional(),
     group_id: zod_1.z.string().optional(),
     dominant_geos: zod_1.z.array(zod_1.z.string()).optional(),
     requested_by: zod_1.z
@@ -105,20 +106,24 @@ exports.zodGeoSelectionSchema = zod_1.z.object({
     updated_at: zod_1.z.number().optional(), // last update date
 });
 exports.zodDataSchema = zod_1.z.object({
-    // Filterable fields
-    text_geo: zod_1.z.array(zod_1.z.string()),
-    timestamp: zod_1.z.number(),
-    platform: zod_1.z.enum(exports.platformsList),
-    source_region: zod_1.z.string().optional(),
     _id: zod_1.z.string().optional(),
+    timestamp: zod_1.z.number().optional(),
+    platform: zod_1.z.enum(exports.platformsList).optional(),
+    source_region_id: zod_1.z.string().optional(),
+    source_region_title: zod_1.z.string().optional(),
+    source_dominant_geos: zod_1.z.array(zod_1.z.string()).optional(),
     source_id: zod_1.z.string().or(zod_1.z.number()),
-    source_public_id: zod_1.z.string().or(zod_1.z.number()),
-    source_name: zod_1.z.string(),
-    platform_id: zod_1.z.string().or(zod_1.z.number()),
-    original_text_id: zod_1.z.string().or(zod_1.z.number()),
-    original_text: zod_1.z.string().optional(),
-    translated_text: zod_1.z.string().optional(),
-    language: zod_1.z.string().optional(),
+    source_title: zod_1.z.string().optional(),
+    source_url: zod_1.z.string().url().optional(),
+    source_group_id: zod_1.z.string().optional(),
+    data_id: zod_1.z.string().or(zod_1.z.number()).optional(),
+    data_geo: zod_1.z.array(zod_1.z.string()).optional(),
+    data_text: zod_1.z.string().optional(),
+    data_url: zod_1.z.string().url().optional(),
+    data_original_type: zod_1.z.string().optional(),
+    data_language: zod_1.z.string().optional(),
+    data_sentiment: zod_1.z.string().optional(),
+    data_timestamp: zod_1.z.number().optional(),
     is_reply: zod_1.z.boolean().optional(),
     reply_to_message_id: zod_1.z.string().or(zod_1.z.number()).optional(),
     metadata: zod_1.z.record(zod_1.z.string(), zod_1.z.any()).optional(),
@@ -132,11 +137,14 @@ exports.zodDataSchema = zod_1.z.object({
     }))
         .optional(),
     author: zod_1.z.string().optional(),
-    replies: zod_1.z.any().optional(),
-    entities: zod_1.z.any().optional(),
     author_username: zod_1.z.string().optional(),
-    author_id: zod_1.z.string().optional(),
-    group_id: zod_1.z.string().optional(), // e.g., 'default_pipeline'
+    replies: zod_1.z.any().optional(),
+    author_id: zod_1.z.string().optional(), // e.g., author ID -> get from post
+    // translated_text: z.string().optional(), // translated text if available - most of the time it will be translated to English
+    // entities: z.any().optional(), // array of reply texts or IDs
+    // source_public_id: z.string().or(z.number()), // e.g., 'telegram:1234567890' (message_id)
+    // platform_id: z.string().or(z.number()), // channel_id reference to Source source_id
+    // original_text_id: z.string().or(z.number()), // message_id
 });
 // Helper function to get allowed entities for a platform
 const getAllowedEntitiesForPlatform = (platform) => {

package/index.ts

@@ -2,36 +2,40 @@
 export * from './types';
 
 import { Schema } from 'mongoose';
-import { 
-  generalStatusList, 
-  platformsList, 
+import {
+  generalStatusList,
+  platformsList,
   entityTypesList,
-  sourceStatusList 
+  sourceStatusList,
 } from './types';
 
 export const MongoDataSchema = new Schema(
   {
-    source_id: { type: Schema.Types.Mixed, required: true }, // reference to Source _id
-    source_public_id: { type: Schema.Types.Mixed }, // e.g., 'telegram:1234567890' (message_id)
-    source_name: { type: String, required: true }, // e.g., 'Telegram Channel Name'
-    platform_id: { type: Schema.Types.Mixed }, // channel_id reference to Source source_id
+    timestamp: { type: Number }, // INDEX - date in milliseconds
     platform: {
       type: String,
       enum: platformsList,
-      required: true,
     }, // e.g., 'telegram', 'facebook'
-    original_text_id: { type: Schema.Types.Mixed }, // message_id
-    original_text: { type: String }, // original text content
-    translated_text: { type: String }, // translated text if available
-    timestamp: { type: Number, required: true }, // date in milliseconds
-    language: { type: String },
-    text_geo: { type: [String], default: [] }, // e.g., ["sinjil", "ramallah", "west bank"]
-    source_geo: { type: String }, // e.g., 'hebron'
+    source_region_id: { type: String }, // e.g., 'hebron'
+    source_region_title: { type: String }, // e.g., 'hebron'
+    source_dominant_geos: { type: [String], default: [] }, // e.g., ['hebron', 'west bank']
+    source_id: { type: Schema.Types.Mixed, required: true }, // INDEX - reference to Source _id
+    source_title: { type: String }, // e.g., 'Telegram Channel Name'
+    source_url: { type: String }, // e.g., 'https://t.me/telegram_channel_name'
+    source_group_id: { type: String }, // e.g., source group ID
+    data_id: { type: Schema.Types.Mixed }, // INDEX - original text ID (e.g., message_id)
+    data_geo: { type: [String], default: [] }, // INDEX - e.g., ["sinjil", "ramallah", "west bank"]
+    data_text: { type: String }, // processed text content
+    data_url: { type: String }, // original text URL if available
+    data_original_type: { type: String }, // e.g., 'post', 'comment', 'reply', 'video', 'image'
+    data_language: { type: String }, // detected language of the text
+    data_sentiment: { type: String }, // sentiment analysis result
+    data_timestamp: { type: Number }, // original post timestamp if different from ingestion timestamp
     is_reply: { type: Boolean }, // true if this text is a reply to another text
     reply_to_message_id: { type: Schema.Types.Mixed },
     metadata: { type: Object }, // platform-specific fields
-    created_at: { type: Number, default: Date.now },
-    updated_at: { type: Number, default: Date.now },
+    created_at: { type: Number, default: Date.now, required: true },
+    updated_at: { type: Number, default: Date.now, required: true },
     media: {
       type: [
         {
@@ -42,12 +46,9 @@ export const MongoDataSchema = new Schema(
       ],
     }, // media attachments
     author: { type: String }, // e.g., author name or ID
-    replies: { type: Schema.Types.Mixed }, // array of reply texts or IDs
-    entities: { type: Schema.Types.Mixed }, // array of entities
     author_username: { type: String }, // e.g., author username
+    replies: { type: Schema.Types.Mixed }, // array of reply texts or IDs
     author_id: { type: String }, // e.g., author ID
-    pipeline_name: { type: String }, // e.g., 'default_pipeline'
-    source_region: { type: String }, // e.g., "New York", "California"
   },
   {
     versionKey: false,
@@ -69,7 +70,6 @@ export const MongoRegionSchema = new Schema(
       default: 'active',
     },
     notes: { type: String }, // internal notes about the region
-
   },
   {
     versionKey: false,
@@ -133,6 +133,7 @@ export const MongoSourceSchema = new Schema(
     created_at: { type: Number, default: Date.now },
     updated_at: { type: Number, default: Date.now },
     region_id: { type: String }, // e.g., region ID
+    region_title: { type: String }, // e.g., region title
     group_id: { type: String }, // e.g., source group ID
     dominant_geos: { type: [String], default: [] }, // e.g., dominant geos associated with this source
     requested_by: {
@@ -146,105 +147,6 @@ export const MongoSourceSchema = new Schema(
       default: undefined,
     }, // user who requested this source
     notes: { type: String }, // internal notes about the source
-
-  },
-  {
-    versionKey: false,
-    toJSON: { virtuals: true },
-    toObject: { virtuals: true },
-  }
-);
-
-export const MongoQuerySchema = new Schema(
-  {
-    title: { type: String }, // e.g., 'Search Query'
-    query: { type: String }, // search query includes geo information where it should be extracted with LLM
-    sources: { type: [String] }, // array of source _id strings
-    geos: { type: [String], required: true }, // array of geo strings
-    geos_ids: { type: [String], required: true }, // array of geo _id strings
-    time_range: {
-      type: Schema.Types.Mixed, // can be relative or absolute time range
-      required: true,
-    },
-    user_id: { type: String, required: true }, // user identifier
-    user_instructions: { type: String }, // optional user instructions for the query
-    user_time_zone: { type: String }, // user's timezone
-    timestamp: { type: Number, default: Date.now }, // creation date
-  },
-  {
-    versionKey: false,
-    toJSON: { virtuals: true },
-    toObject: { virtuals: true },
-  }
-);
-
-export const MongoUserSettingsSchema = new Schema(
-  {
-    sources: { type: [String] }, // array of source _id strings
-    geos: { type: [String], default: [] }, // array of geo strings
-    time_range: {
-      type: Schema.Types.Mixed,
-    },
-    user_id: { type: String, required: true },
-    thread_id: { type: String },
-    instructions: { type: String }, // optional user instructions for the query
-    has_jobs_access: { type: Boolean, default: false }, // true if the user has access to jobs
-    active_jobs_limit: { type: Number, default: 5 }, // maximum number of active jobs allowed
-    created_at: { type: Date, default: Date.now }, // creation date
-    updated_at: { type: Date, default: Date.now }, // last update date
-    status: {
-      type: String,
-      enum: generalStatusList,
-      default: 'active',
-    }, // status of the user
-  },
-  {
-    versionKey: false,
-    toJSON: { virtuals: true },
-    toObject: { virtuals: true },
-  }
-);
-
-export const MongoConversationSchema = new Schema(
-  {
-    user_id: { type: String, required: true }, // user identifier
-    title: { type: String }, // optional title for the conversation
-    thread_id: { type: String }, // thread ID for the conversation
-    status: {
-      type: String,
-      enum: generalStatusList,
-      default: 'active',
-    }, // status of the conversation
-    created_at: { type: Date, default: Date.now }, // creation date
-    updated_at: { type: Date, default: Date.now }, // last update date
-    is_job: { type: Boolean, default: false }, // true if this is a job conversation
-  },
-  {
-    versionKey: false,
-    toJSON: { virtuals: true },
-    toObject: { virtuals: true },
-  }
-);
-
-export const MongoJobSchema = new Schema(
-  {
-    user_id: { type: String, required: true }, // user identifier
-    job_name: { type: String, required: true }, // name of the automation job
-    job_description: { type: String }, // description of the job
-    status: {
-      type: String,
-      enum: generalStatusList,
-      default: 'active',
-    }, // status of the job
-    created_at: { type: Number, default: Date.now }, // creation date
-    updated_at: { type: Number, default: Date.now }, // last update date
-    schedule: { type: String }, // cron schedule for the job
-    schedule_text: { type: String }, // cron schedule in free-text format
-    active_until: { type: Number || undefined }, // optional date until which the job is active
-    query: { type: Schema.Types.Mixed, required: true }, // query associated with the job
-    conversation_id: { type: String }, // optional conversation ID for the job
-    thread_id: { type: String }, // thread ID for the job
-    time_zone: { type: String }, // optional timezone for the job
   },
   {
     versionKey: false,

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "harvester_sdk",
-  "version": "1.0.6",
+  "version": "1.0.8",
   "description": "SDK for interacting with the Harvester API",
   "main": "dist/index.js",
   "types": "dist/index.d.ts",

package/types.ts

@@ -77,6 +77,7 @@ export const zodSourceSchema = z.object({
   created_at: z.number().optional(),
   updated_at: z.number().optional(),
   region_id: z.string().optional(), // e.g., region ID // INDEX
+  region_title: z.string().optional(), // e.g., region title
   group_id: z.string().optional(), // e.g., source group ID // INDEX
   dominant_geos: z.array(z.string()).optional(), // e.g., dominant geos associated with this source
   requested_by: z
@@ -113,24 +114,28 @@ export const zodGeoSelectionSchema = z.object({
 });
 
 export const zodDataSchema = z.object({
-  // Filterable fields
-  text_geo: z.array(z.string()), // INDEX - e.g., ["sinjil", "ramallah", "west bank"]
-  timestamp: z.number(), // INDEX - date in milliseconds - e.g., 1751210833000
-  platform: z.enum(platformsList), // e.g., 'telegram', 'facebook'
-  source_region: z.string().optional(), // e.g., 'hebron'
   _id: z.string().optional(),
-  source_id: z.string().or(z.number()), // INDEX - reference to Source _id (e.g., '60c72b2f9b1e8d3f4c8b4567')
-  source_public_id: z.string().or(z.number()), // e.g., 'telegram:1234567890' (message_id)
-  source_name: z.string(), // e.g., 'Telegram Channel Name'
-  platform_id: z.string().or(z.number()), // channel_id reference to Source source_id
-  original_text_id: z.string().or(z.number()), // message_id
-  original_text: z.string().optional(), // original text content
-  translated_text: z.string().optional(), // translated text if available - most of the time it will be translated to English
-  language: z.string().optional(),
-  is_reply: z.boolean().optional(), // true if this text is a reply to another text
-  reply_to_message_id: z.string().or(z.number()).optional(),
-  metadata: z.record(z.string(), z.any()).optional(), // platform-specific fields
-  created_at: z.number(),
+  timestamp: z.number().optional(), // INDEX - date in milliseconds - e.g., 1751210833000
+  platform: z.enum(platformsList).optional(), // e.g., 'telegram', 'facebook'
+  source_region_id: z.string().optional(), // e.g., 'hebron' -> get from source object
+  source_region_title: z.string().optional(), // e.g., 'hebron' -> get from source object
+  source_dominant_geos: z.array(z.string()).optional(), // e.g., ['hebron', 'west bank'] -> get from source object
+  source_id: z.string().or(z.number()), // INDEX - reference to Source _id (e.g., '60c72b2f9b1e8d3f4c8b4567') -> get from source object
+  source_title: z.string().optional(), // e.g., 'Telegram Channel Name' -> get from source object
+  source_url: z.string().url().optional(), // e.g., 'https://t.me/telegram_channel_name' -> get from source object
+  source_group_id: z.string().optional(), // e.g., 'default_pipeline' -> get from source object
+  data_id: z.string().or(z.number()).optional(), // INDEX - original text ID (e.g., message_id) -> get from post
+  data_geo: z.array(z.string()).optional(), // INDEX - e.g., ["sinjil", "ramallah", "west bank"] -> get from processor
+  data_text: z.string().optional(), // processed text content -> get from post or processor
+  data_url: z.string().url().optional(), // original text URL if available -> get from post
+  data_original_type: z.string().optional(), // e.g., 'post', 'comment', 'reply', 'video', 'image' -> get from post
+  data_language: z.string().optional(), // detected language of the text -> get from post or processor
+  data_sentiment: z.string().optional(), // sentiment analysis result -> get from processor
+  data_timestamp: z.number().optional(), // original post timestamp if different from ingestion timestamp -> get from post
+  is_reply: z.boolean().optional(), // true if this text is a reply to another text -> get from post
+  reply_to_message_id: z.string().or(z.number()).optional(), // ID of the message this is a reply to -> get from post
+  metadata: z.record(z.string(), z.any()).optional(), // platform-specific fields -> get from post
+  created_at: z.number(), 
   updated_at: z.number(),
   media: z
     .array(
@@ -140,22 +145,18 @@ export const zodDataSchema = z.object({
         caption: z.string().optional(),
       })
     )
-    .optional(), // media attachments
-  author: z.string().optional(), // e.g., author name or ID
-  replies: z.any().optional(), // array of reply texts or IDs
-  entities: z.any().optional(), // array of reply texts or IDs
-  author_username: z.string().optional(), // e.g., author username
-  author_id: z.string().optional(), // e.g., author ID
-  group_id: z.string().optional(), // e.g., 'default_pipeline'
+    .optional(), // media attachments -> get from post
+  author: z.string().optional(), // e.g., author name or ID -> get from post
+  author_username: z.string().optional(), // e.g., author username -> get from post
+  replies: z.any().optional(), // array of reply texts or IDs -> get from post
+  author_id: z.string().optional(), // e.g., author ID -> get from post
+  // translated_text: z.string().optional(), // translated text if available - most of the time it will be translated to English
+  // entities: z.any().optional(), // array of reply texts or IDs
+  // source_public_id: z.string().or(z.number()), // e.g., 'telegram:1234567890' (message_id)
+  // platform_id: z.string().or(z.number()), // channel_id reference to Source source_id
+  // original_text_id: z.string().or(z.number()), // message_id
 });
 
-/**
- * 'approved' - active and approved sources,
- * 'back_to_business' - sources that were paused and now resumed,
- * 'pending' - sources that are pending approval,
- * 'inactive' - sources that are inactive,
- * 'in_review' - sources that are under review
- */
 export type RegionType = z.infer<typeof zodRegionSchema>;
 export type SourceGroupType = z.infer<typeof zodSourceGroupSchema>;
 export type SourceType = z.infer<typeof zodSourceSchema>;