harvester_sdk 1.0.5 → 1.0.7

package/dist/index.d.ts

@@ -36,11 +36,8 @@ export declare const MongoDataSchema: Schema<any, import("mongoose").Model<any,
 }, {
     created_at: number;
     updated_at: number;
-    platform: "telegram" | "facebook" | "instagram" | "tiktok" | "website";
-    timestamp: number;
-    text_geo: string[];
     source_id: any;
-    source_name: string;
+    data_geo: string[];
     media: import("mongoose").Types.DocumentArray<{
         type?: "image" | "video" | "audio" | "link" | null | undefined;
         url?: string | null | undefined;
@@ -54,31 +51,32 @@ export declare const MongoDataSchema: Schema<any, import("mongoose").Model<any,
         url?: string | null | undefined;
         caption?: string | null | undefined;
     }>;
-    language?: string | null | undefined;
+    platform?: "telegram" | "facebook" | "instagram" | "tiktok" | "website" | null | undefined;
     metadata?: any;
-    source_region?: string | null | undefined;
-    source_public_id?: any;
-    platform_id?: any;
-    original_text_id?: any;
-    original_text?: string | null | undefined;
-    translated_text?: string | null | undefined;
+    group_id?: string | null | undefined;
+    timestamp?: number | null | undefined;
+    source_region_id?: string | null | undefined;
+    source_region_title?: string | null | undefined;
+    source_title?: string | null | undefined;
+    source_url?: string | null | undefined;
+    data_id?: any;
+    data_text?: string | null | undefined;
+    data_url?: string | null | undefined;
+    data_original_type?: string | null | undefined;
+    data_language?: string | null | undefined;
+    data_sentiment?: string | null | undefined;
+    data_timestamp?: number | null | undefined;
     is_reply?: boolean | null | undefined;
     reply_to_message_id?: any;
     author?: string | null | undefined;
-    replies?: any;
-    entities?: any;
     author_username?: string | null | undefined;
+    replies?: any;
     author_id?: string | null | undefined;
-    source_geo?: string | null | undefined;
-    pipeline_name?: string | null | undefined;
 }, import("mongoose").Document<unknown, {}, import("mongoose").FlatRecord<{
     created_at: number;
     updated_at: number;
-    platform: "telegram" | "facebook" | "instagram" | "tiktok" | "website";
-    timestamp: number;
-    text_geo: string[];
     source_id: any;
-    source_name: string;
+    data_geo: string[];
     media: import("mongoose").Types.DocumentArray<{
         type?: "image" | "video" | "audio" | "link" | null | undefined;
         url?: string | null | undefined;
@@ -92,31 +90,32 @@ export declare const MongoDataSchema: Schema<any, import("mongoose").Model<any,
         url?: string | null | undefined;
         caption?: string | null | undefined;
     }>;
-    language?: string | null | undefined;
+    platform?: "telegram" | "facebook" | "instagram" | "tiktok" | "website" | null | undefined;
     metadata?: any;
-    source_region?: string | null | undefined;
-    source_public_id?: any;
-    platform_id?: any;
-    original_text_id?: any;
-    original_text?: string | null | undefined;
-    translated_text?: string | null | undefined;
+    group_id?: string | null | undefined;
+    timestamp?: number | null | undefined;
+    source_region_id?: string | null | undefined;
+    source_region_title?: string | null | undefined;
+    source_title?: string | null | undefined;
+    source_url?: string | null | undefined;
+    data_id?: any;
+    data_text?: string | null | undefined;
+    data_url?: string | null | undefined;
+    data_original_type?: string | null | undefined;
+    data_language?: string | null | undefined;
+    data_sentiment?: string | null | undefined;
+    data_timestamp?: number | null | undefined;
     is_reply?: boolean | null | undefined;
     reply_to_message_id?: any;
     author?: string | null | undefined;
-    replies?: any;
-    entities?: any;
     author_username?: string | null | undefined;
+    replies?: any;
     author_id?: string | null | undefined;
-    source_geo?: string | null | undefined;
-    pipeline_name?: string | null | undefined;
 }>, {}> & import("mongoose").FlatRecord<{
     created_at: number;
     updated_at: number;
-    platform: "telegram" | "facebook" | "instagram" | "tiktok" | "website";
-    timestamp: number;
-    text_geo: string[];
     source_id: any;
-    source_name: string;
+    data_geo: string[];
     media: import("mongoose").Types.DocumentArray<{
         type?: "image" | "video" | "audio" | "link" | null | undefined;
         url?: string | null | undefined;
@@ -130,23 +129,27 @@ export declare const MongoDataSchema: Schema<any, import("mongoose").Model<any,
         url?: string | null | undefined;
         caption?: string | null | undefined;
     }>;
-    language?: string | null | undefined;
+    platform?: "telegram" | "facebook" | "instagram" | "tiktok" | "website" | null | undefined;
     metadata?: any;
-    source_region?: string | null | undefined;
-    source_public_id?: any;
-    platform_id?: any;
-    original_text_id?: any;
-    original_text?: string | null | undefined;
-    translated_text?: string | null | undefined;
+    group_id?: string | null | undefined;
+    timestamp?: number | null | undefined;
+    source_region_id?: string | null | undefined;
+    source_region_title?: string | null | undefined;
+    source_title?: string | null | undefined;
+    source_url?: string | null | undefined;
+    data_id?: any;
+    data_text?: string | null | undefined;
+    data_url?: string | null | undefined;
+    data_original_type?: string | null | undefined;
+    data_language?: string | null | undefined;
+    data_sentiment?: string | null | undefined;
+    data_timestamp?: number | null | undefined;
     is_reply?: boolean | null | undefined;
     reply_to_message_id?: any;
     author?: string | null | undefined;
-    replies?: any;
-    entities?: any;
     author_username?: string | null | undefined;
+    replies?: any;
     author_id?: string | null | undefined;
-    source_geo?: string | null | undefined;
-    pipeline_name?: string | null | undefined;
 }> & {
     _id: import("mongoose").Types.ObjectId;
 } & {
@@ -241,7 +244,6 @@ export declare const MongoSourceSchema: Schema<any, import("mongoose").Model<any
         virtuals: true;
     };
 }, {
-    name: string;
     created_at: number;
     updated_at: number;
     status: "active" | "pending" | "inactive" | "requested" | "discovered" | "deleted";
@@ -267,7 +269,6 @@ export declare const MongoSourceSchema: Schema<any, import("mongoose").Model<any
         requested_at?: number | null | undefined;
     } | null | undefined;
 }, import("mongoose").Document<unknown, {}, import("mongoose").FlatRecord<{
-    name: string;
     created_at: number;
     updated_at: number;
     status: "active" | "pending" | "inactive" | "requested" | "discovered" | "deleted";
@@ -293,7 +294,6 @@ export declare const MongoSourceSchema: Schema<any, import("mongoose").Model<any
         requested_at?: number | null | undefined;
     } | null | undefined;
 }>, {}> & import("mongoose").FlatRecord<{
-    name: string;
     created_at: number;
     updated_at: number;
     status: "active" | "pending" | "inactive" | "requested" | "discovered" | "deleted";
@@ -323,193 +323,6 @@ export declare const MongoSourceSchema: Schema<any, import("mongoose").Model<any
 } & {
     __v: number;
 }>;
-export declare const MongoQuerySchema: Schema<any, import("mongoose").Model<any, any, any, any, any, any>, {}, {}, {}, {}, {
-    versionKey: false;
-    toJSON: {
-        virtuals: true;
-    };
-    toObject: {
-        virtuals: true;
-    };
-}, {
-    user_id: string;
-    timestamp: number;
-    geos: string[];
-    sources: string[];
-    geos_ids: string[];
-    time_range: any;
-    title?: string | null | undefined;
-    query?: string | null | undefined;
-    user_instructions?: string | null | undefined;
-    user_time_zone?: string | null | undefined;
-}, import("mongoose").Document<unknown, {}, import("mongoose").FlatRecord<{
-    user_id: string;
-    timestamp: number;
-    geos: string[];
-    sources: string[];
-    geos_ids: string[];
-    time_range: any;
-    title?: string | null | undefined;
-    query?: string | null | undefined;
-    user_instructions?: string | null | undefined;
-    user_time_zone?: string | null | undefined;
-}>, {}> & import("mongoose").FlatRecord<{
-    user_id: string;
-    timestamp: number;
-    geos: string[];
-    sources: string[];
-    geos_ids: string[];
-    time_range: any;
-    title?: string | null | undefined;
-    query?: string | null | undefined;
-    user_instructions?: string | null | undefined;
-    user_time_zone?: string | null | undefined;
-}> & {
-    _id: import("mongoose").Types.ObjectId;
-} & {
-    __v: number;
-}>;
-export declare const MongoUserSettingsSchema: Schema<any, import("mongoose").Model<any, any, any, any, any, any>, {}, {}, {}, {}, {
-    versionKey: false;
-    toJSON: {
-        virtuals: true;
-    };
-    toObject: {
-        virtuals: true;
-    };
-}, {
-    created_at: NativeDate;
-    updated_at: NativeDate;
-    status: "active" | "inactive" | "deleted";
-    user_id: string;
-    geos: string[];
-    sources: string[];
-    has_jobs_access: boolean;
-    active_jobs_limit: number;
-    time_range?: any;
-    thread_id?: string | null | undefined;
-    instructions?: string | null | undefined;
-}, import("mongoose").Document<unknown, {}, import("mongoose").FlatRecord<{
-    created_at: NativeDate;
-    updated_at: NativeDate;
-    status: "active" | "inactive" | "deleted";
-    user_id: string;
-    geos: string[];
-    sources: string[];
-    has_jobs_access: boolean;
-    active_jobs_limit: number;
-    time_range?: any;
-    thread_id?: string | null | undefined;
-    instructions?: string | null | undefined;
-}>, {}> & import("mongoose").FlatRecord<{
-    created_at: NativeDate;
-    updated_at: NativeDate;
-    status: "active" | "inactive" | "deleted";
-    user_id: string;
-    geos: string[];
-    sources: string[];
-    has_jobs_access: boolean;
-    active_jobs_limit: number;
-    time_range?: any;
-    thread_id?: string | null | undefined;
-    instructions?: string | null | undefined;
-}> & {
-    _id: import("mongoose").Types.ObjectId;
-} & {
-    __v: number;
-}>;
-export declare const MongoConversationSchema: Schema<any, import("mongoose").Model<any, any, any, any, any, any>, {}, {}, {}, {}, {
-    versionKey: false;
-    toJSON: {
-        virtuals: true;
-    };
-    toObject: {
-        virtuals: true;
-    };
-}, {
-    created_at: NativeDate;
-    updated_at: NativeDate;
-    status: "active" | "inactive" | "deleted";
-    user_id: string;
-    is_job: boolean;
-    title?: string | null | undefined;
-    thread_id?: string | null | undefined;
-}, import("mongoose").Document<unknown, {}, import("mongoose").FlatRecord<{
-    created_at: NativeDate;
-    updated_at: NativeDate;
-    status: "active" | "inactive" | "deleted";
-    user_id: string;
-    is_job: boolean;
-    title?: string | null | undefined;
-    thread_id?: string | null | undefined;
-}>, {}> & import("mongoose").FlatRecord<{
-    created_at: NativeDate;
-    updated_at: NativeDate;
-    status: "active" | "inactive" | "deleted";
-    user_id: string;
-    is_job: boolean;
-    title?: string | null | undefined;
-    thread_id?: string | null | undefined;
-}> & {
-    _id: import("mongoose").Types.ObjectId;
-} & {
-    __v: number;
-}>;
-export declare const MongoJobSchema: Schema<any, import("mongoose").Model<any, any, any, any, any, any>, {}, {}, {}, {}, {
-    versionKey: false;
-    toJSON: {
-        virtuals: true;
-    };
-    toObject: {
-        virtuals: true;
-    };
-}, {
-    created_at: number;
-    updated_at: number;
-    status: "active" | "inactive" | "deleted";
-    user_id: string;
-    query: any;
-    job_name: string;
-    thread_id?: string | null | undefined;
-    job_description?: string | null | undefined;
-    schedule?: string | null | undefined;
-    schedule_text?: string | null | undefined;
-    active_until?: number | null | undefined;
-    conversation_id?: string | null | undefined;
-    time_zone?: string | null | undefined;
-}, import("mongoose").Document<unknown, {}, import("mongoose").FlatRecord<{
-    created_at: number;
-    updated_at: number;
-    status: "active" | "inactive" | "deleted";
-    user_id: string;
-    query: any;
-    job_name: string;
-    thread_id?: string | null | undefined;
-    job_description?: string | null | undefined;
-    schedule?: string | null | undefined;
-    schedule_text?: string | null | undefined;
-    active_until?: number | null | undefined;
-    conversation_id?: string | null | undefined;
-    time_zone?: string | null | undefined;
-}>, {}> & import("mongoose").FlatRecord<{
-    created_at: number;
-    updated_at: number;
-    status: "active" | "inactive" | "deleted";
-    user_id: string;
-    query: any;
-    job_name: string;
-    thread_id?: string | null | undefined;
-    job_description?: string | null | undefined;
-    schedule?: string | null | undefined;
-    schedule_text?: string | null | undefined;
-    active_until?: number | null | undefined;
-    conversation_id?: string | null | undefined;
-    time_zone?: string | null | undefined;
-}> & {
-    _id: import("mongoose").Types.ObjectId;
-} & {
-    __v: number;
-}>;
 export declare const MongoGeoSchema: Schema<any, import("mongoose").Model<any, any, any, any, any, any>, {}, {}, {}, {}, {
     versionKey: false;
     toJSON: {

package/dist/index.js

@@ -14,33 +14,35 @@ var __exportStar = (this && this.__exportStar) || function(m, exports) {
     for (var p in m) if (p !== "default" && !Object.prototype.hasOwnProperty.call(exports, p)) __createBinding(exports, m, p);
 };
 Object.defineProperty(exports, "__esModule", { value: true });
-exports.MongoApiKeySchema = exports.MongoGeoSelectionSchema = exports.MongoGeoSchema = exports.MongoJobSchema = exports.MongoConversationSchema = exports.MongoUserSettingsSchema = exports.MongoQuerySchema = exports.MongoSourceSchema = exports.MongoSourceGroupSchema = exports.MongoRegionSchema = exports.MongoDataSchema = void 0;
+exports.MongoApiKeySchema = exports.MongoGeoSelectionSchema = exports.MongoGeoSchema = exports.MongoSourceSchema = exports.MongoSourceGroupSchema = exports.MongoRegionSchema = exports.MongoDataSchema = void 0;
 // Export types for consumers
 __exportStar(require("./types"), exports);
 const mongoose_1 = require("mongoose");
 const types_1 = require("./types");
 exports.MongoDataSchema = new mongoose_1.Schema({
-    source_id: { type: mongoose_1.Schema.Types.Mixed, required: true },
-    source_public_id: { type: mongoose_1.Schema.Types.Mixed },
-    source_name: { type: String, required: true },
-    platform_id: { type: mongoose_1.Schema.Types.Mixed },
+    timestamp: { type: Number },
     platform: {
         type: String,
         enum: types_1.platformsList,
-        required: true,
     },
-    original_text_id: { type: mongoose_1.Schema.Types.Mixed },
-    original_text: { type: String },
-    translated_text: { type: String },
-    timestamp: { type: Number, required: true },
-    language: { type: String },
-    text_geo: { type: [String], default: [] },
-    source_geo: { type: String },
+    source_region_id: { type: String },
+    source_region_title: { type: String },
+    source_id: { type: mongoose_1.Schema.Types.Mixed, required: true },
+    source_title: { type: String },
+    source_url: { type: String },
+    data_id: { type: mongoose_1.Schema.Types.Mixed },
+    data_geo: { type: [String], default: [] },
+    data_text: { type: String },
+    data_url: { type: String },
+    data_original_type: { type: String },
+    data_language: { type: String },
+    data_sentiment: { type: String },
+    data_timestamp: { type: Number },
     is_reply: { type: Boolean },
     reply_to_message_id: { type: mongoose_1.Schema.Types.Mixed },
     metadata: { type: Object },
-    created_at: { type: Number, default: Date.now },
-    updated_at: { type: Number, default: Date.now },
+    created_at: { type: Number, default: Date.now, required: true },
+    updated_at: { type: Number, default: Date.now, required: true },
     media: {
         type: [
             {
@@ -51,12 +53,10 @@ exports.MongoDataSchema = new mongoose_1.Schema({
         ],
     },
     author: { type: String },
-    replies: { type: mongoose_1.Schema.Types.Mixed },
-    entities: { type: mongoose_1.Schema.Types.Mixed },
     author_username: { type: String },
+    replies: { type: mongoose_1.Schema.Types.Mixed },
     author_id: { type: String },
-    pipeline_name: { type: String },
-    source_region: { type: String }, // e.g., "New York", "California"
+    group_id: { type: String }, // e.g., source group ID
 }, {
     versionKey: false,
     toJSON: { virtuals: true },
@@ -115,7 +115,6 @@ exports.MongoSourceSchema = new mongoose_1.Schema({
         required: true,
     },
     public_id: { type: mongoose_1.Schema.Types.Mixed },
-    name: { type: String, required: true },
     url: { type: String },
     description: { type: String },
     language: { type: String },
@@ -148,88 +147,6 @@ exports.MongoSourceSchema = new mongoose_1.Schema({
     toJSON: { virtuals: true },
     toObject: { virtuals: true },
 });
-exports.MongoQuerySchema = new mongoose_1.Schema({
-    title: { type: String },
-    query: { type: String },
-    sources: { type: [String] },
-    geos: { type: [String], required: true },
-    geos_ids: { type: [String], required: true },
-    time_range: {
-        type: mongoose_1.Schema.Types.Mixed,
-        required: true,
-    },
-    user_id: { type: String, required: true },
-    user_instructions: { type: String },
-    user_time_zone: { type: String },
-    timestamp: { type: Number, default: Date.now }, // creation date
-}, {
-    versionKey: false,
-    toJSON: { virtuals: true },
-    toObject: { virtuals: true },
-});
-exports.MongoUserSettingsSchema = new mongoose_1.Schema({
-    sources: { type: [String] },
-    geos: { type: [String], default: [] },
-    time_range: {
-        type: mongoose_1.Schema.Types.Mixed,
-    },
-    user_id: { type: String, required: true },
-    thread_id: { type: String },
-    instructions: { type: String },
-    has_jobs_access: { type: Boolean, default: false },
-    active_jobs_limit: { type: Number, default: 5 },
-    created_at: { type: Date, default: Date.now },
-    updated_at: { type: Date, default: Date.now },
-    status: {
-        type: String,
-        enum: types_1.generalStatusList,
-        default: 'active',
-    }, // status of the user
-}, {
-    versionKey: false,
-    toJSON: { virtuals: true },
-    toObject: { virtuals: true },
-});
-exports.MongoConversationSchema = new mongoose_1.Schema({
-    user_id: { type: String, required: true },
-    title: { type: String },
-    thread_id: { type: String },
-    status: {
-        type: String,
-        enum: types_1.generalStatusList,
-        default: 'active',
-    },
-    created_at: { type: Date, default: Date.now },
-    updated_at: { type: Date, default: Date.now },
-    is_job: { type: Boolean, default: false }, // true if this is a job conversation
-}, {
-    versionKey: false,
-    toJSON: { virtuals: true },
-    toObject: { virtuals: true },
-});
-exports.MongoJobSchema = new mongoose_1.Schema({
-    user_id: { type: String, required: true },
-    job_name: { type: String, required: true },
-    job_description: { type: String },
-    status: {
-        type: String,
-        enum: types_1.generalStatusList,
-        default: 'active',
-    },
-    created_at: { type: Number, default: Date.now },
-    updated_at: { type: Number, default: Date.now },
-    schedule: { type: String },
-    schedule_text: { type: String },
-    active_until: { type: Number || undefined },
-    query: { type: mongoose_1.Schema.Types.Mixed, required: true },
-    conversation_id: { type: String },
-    thread_id: { type: String },
-    time_zone: { type: String }, // optional timezone for the job
-}, {
-    versionKey: false,
-    toJSON: { virtuals: true },
-    toObject: { virtuals: true },
-});
 exports.MongoGeoSchema = new mongoose_1.Schema({
     geo_text: { type: String },
     timestamp: { type: Number },

package/dist/types.d.ts

@@ -78,7 +78,6 @@ export declare const zodSourceSchema: z.ZodObject<{
     platform: z.ZodEnum<["telegram", "facebook", "instagram", "tiktok", "website"]>;
     entity: z.ZodEnum<["profile", "group", "page", "channel", "hashtag", "website"]>;
     public_id: z.ZodOptional<z.ZodUnion<[z.ZodString, z.ZodNumber]>>;
-    name: z.ZodString;
     url: z.ZodOptional<z.ZodString>;
     description: z.ZodOptional<z.ZodString>;
     language: z.ZodOptional<z.ZodString>;
@@ -112,7 +111,6 @@ export declare const zodSourceSchema: z.ZodObject<{
     }>>;
     notes: z.ZodOptional<z.ZodString>;
 }, "strip", z.ZodTypeAny, {
-    name: string;
     status: "active" | "pending" | "inactive" | "requested" | "discovered" | "deleted";
     platform: "telegram" | "facebook" | "instagram" | "tiktok" | "website";
     entity: "website" | "profile" | "group" | "page" | "channel" | "hashtag";
@@ -139,7 +137,6 @@ export declare const zodSourceSchema: z.ZodObject<{
         requested_at?: number | undefined;
     } | undefined;
 }, {
-    name: string;
     status: "active" | "pending" | "inactive" | "requested" | "discovered" | "deleted";
     platform: "telegram" | "facebook" | "instagram" | "tiktok" | "website";
     entity: "website" | "profile" | "group" | "page" | "channel" | "hashtag";
@@ -220,19 +217,22 @@ export declare const zodGeoSelectionSchema: z.ZodObject<{
     geos?: string[] | undefined;
 }>;
 export declare const zodDataSchema: z.ZodObject<{
-    text_geo: z.ZodArray<z.ZodString, "many">;
-    timestamp: z.ZodNumber;
-    platform: z.ZodEnum<["telegram", "facebook", "instagram", "tiktok", "website"]>;
-    source_region: z.ZodOptional<z.ZodString>;
     _id: z.ZodOptional<z.ZodString>;
+    timestamp: z.ZodOptional<z.ZodNumber>;
+    platform: z.ZodOptional<z.ZodEnum<["telegram", "facebook", "instagram", "tiktok", "website"]>>;
+    source_region_id: z.ZodOptional<z.ZodString>;
+    source_region_title: z.ZodOptional<z.ZodString>;
     source_id: z.ZodUnion<[z.ZodString, z.ZodNumber]>;
-    source_public_id: z.ZodUnion<[z.ZodString, z.ZodNumber]>;
-    source_name: z.ZodString;
-    platform_id: z.ZodUnion<[z.ZodString, z.ZodNumber]>;
-    original_text_id: z.ZodUnion<[z.ZodString, z.ZodNumber]>;
-    original_text: z.ZodOptional<z.ZodString>;
-    translated_text: z.ZodOptional<z.ZodString>;
-    language: z.ZodOptional<z.ZodString>;
+    source_title: z.ZodOptional<z.ZodString>;
+    source_url: z.ZodOptional<z.ZodString>;
+    data_id: z.ZodOptional<z.ZodUnion<[z.ZodString, z.ZodNumber]>>;
+    data_geo: z.ZodOptional<z.ZodArray<z.ZodString, "many">>;
+    data_text: z.ZodOptional<z.ZodString>;
+    data_url: z.ZodOptional<z.ZodString>;
+    data_original_type: z.ZodOptional<z.ZodString>;
+    data_language: z.ZodOptional<z.ZodString>;
+    data_sentiment: z.ZodOptional<z.ZodString>;
+    data_timestamp: z.ZodOptional<z.ZodNumber>;
     is_reply: z.ZodOptional<z.ZodBoolean>;
     reply_to_message_id: z.ZodOptional<z.ZodUnion<[z.ZodString, z.ZodNumber]>>;
     metadata: z.ZodOptional<z.ZodRecord<z.ZodString, z.ZodAny>>;
@@ -252,29 +252,31 @@ export declare const zodDataSchema: z.ZodObject<{
         caption?: string | undefined;
     }>, "many">>;
     author: z.ZodOptional<z.ZodString>;
-    replies: z.ZodOptional<z.ZodAny>;
-    entities: z.ZodOptional<z.ZodAny>;
     author_username: z.ZodOptional<z.ZodString>;
+    replies: z.ZodOptional<z.ZodAny>;
     author_id: z.ZodOptional<z.ZodString>;
     group_id: z.ZodOptional<z.ZodString>;
 }, "strip", z.ZodTypeAny, {
     created_at: number;
     updated_at: number;
-    platform: "telegram" | "facebook" | "instagram" | "tiktok" | "website";
-    timestamp: number;
-    text_geo: string[];
     source_id: string | number;
-    source_public_id: string | number;
-    source_name: string;
-    platform_id: string | number;
-    original_text_id: string | number;
     _id?: string | undefined;
-    language?: string | undefined;
+    platform?: "telegram" | "facebook" | "instagram" | "tiktok" | "website" | undefined;
     metadata?: Record<string, any> | undefined;
     group_id?: string | undefined;
-    source_region?: string | undefined;
-    original_text?: string | undefined;
-    translated_text?: string | undefined;
+    timestamp?: number | undefined;
+    source_region_id?: string | undefined;
+    source_region_title?: string | undefined;
+    source_title?: string | undefined;
+    source_url?: string | undefined;
+    data_id?: string | number | undefined;
+    data_geo?: string[] | undefined;
+    data_text?: string | undefined;
+    data_url?: string | undefined;
+    data_original_type?: string | undefined;
+    data_language?: string | undefined;
+    data_sentiment?: string | undefined;
+    data_timestamp?: number | undefined;
     is_reply?: boolean | undefined;
     reply_to_message_id?: string | number | undefined;
     media?: {
@@ -283,28 +285,30 @@ export declare const zodDataSchema: z.ZodObject<{
         caption?: string | undefined;
     }[] | undefined;
     author?: string | undefined;
-    replies?: any;
-    entities?: any;
     author_username?: string | undefined;
+    replies?: any;
     author_id?: string | undefined;
 }, {
     created_at: number;
     updated_at: number;
-    platform: "telegram" | "facebook" | "instagram" | "tiktok" | "website";
-    timestamp: number;
-    text_geo: string[];
     source_id: string | number;
-    source_public_id: string | number;
-    source_name: string;
-    platform_id: string | number;
-    original_text_id: string | number;
     _id?: string | undefined;
-    language?: string | undefined;
+    platform?: "telegram" | "facebook" | "instagram" | "tiktok" | "website" | undefined;
     metadata?: Record<string, any> | undefined;
     group_id?: string | undefined;
-    source_region?: string | undefined;
-    original_text?: string | undefined;
-    translated_text?: string | undefined;
+    timestamp?: number | undefined;
+    source_region_id?: string | undefined;
+    source_region_title?: string | undefined;
+    source_title?: string | undefined;
+    source_url?: string | undefined;
+    data_id?: string | number | undefined;
+    data_geo?: string[] | undefined;
+    data_text?: string | undefined;
+    data_url?: string | undefined;
+    data_original_type?: string | undefined;
+    data_language?: string | undefined;
+    data_sentiment?: string | undefined;
+    data_timestamp?: number | undefined;
     is_reply?: boolean | undefined;
     reply_to_message_id?: string | number | undefined;
     media?: {
@@ -313,18 +317,10 @@ export declare const zodDataSchema: z.ZodObject<{
         caption?: string | undefined;
     }[] | undefined;
     author?: string | undefined;
-    replies?: any;
-    entities?: any;
     author_username?: string | undefined;
+    replies?: any;
     author_id?: string | undefined;
 }>;
-/**
- * 'approved' - active and approved sources,
- * 'back_to_business' - sources that were paused and now resumed,
- * 'pending' - sources that are pending approval,
- * 'inactive' - sources that are inactive,
- * 'in_review' - sources that are under review
- */
 export type RegionType = z.infer<typeof zodRegionSchema>;
 export type SourceGroupType = z.infer<typeof zodSourceGroupSchema>;
 export type SourceType = z.infer<typeof zodSourceSchema>;

package/dist/types.js

@@ -62,7 +62,6 @@ exports.zodSourceSchema = zod_1.z.object({
     platform: zod_1.z.enum(exports.platformsList),
     entity: zod_1.z.enum(exports.entityTypesList),
     public_id: zod_1.z.string().or(zod_1.z.number()).optional(),
-    name: zod_1.z.string(),
     url: zod_1.z.string().url().optional(),
     description: zod_1.z.string().optional(),
     language: zod_1.z.string().optional(),
@@ -106,20 +105,22 @@ exports.zodGeoSelectionSchema = zod_1.z.object({
     updated_at: zod_1.z.number().optional(), // last update date
 });
 exports.zodDataSchema = zod_1.z.object({
-    // Filterable fields
-    text_geo: zod_1.z.array(zod_1.z.string()),
-    timestamp: zod_1.z.number(),
-    platform: zod_1.z.enum(exports.platformsList),
-    source_region: zod_1.z.string().optional(),
     _id: zod_1.z.string().optional(),
+    timestamp: zod_1.z.number().optional(),
+    platform: zod_1.z.enum(exports.platformsList).optional(),
+    source_region_id: zod_1.z.string().optional(),
+    source_region_title: zod_1.z.string().optional(),
     source_id: zod_1.z.string().or(zod_1.z.number()),
-    source_public_id: zod_1.z.string().or(zod_1.z.number()),
-    source_name: zod_1.z.string(),
-    platform_id: zod_1.z.string().or(zod_1.z.number()),
-    original_text_id: zod_1.z.string().or(zod_1.z.number()),
-    original_text: zod_1.z.string().optional(),
-    translated_text: zod_1.z.string().optional(),
-    language: zod_1.z.string().optional(),
+    source_title: zod_1.z.string().optional(),
+    source_url: zod_1.z.string().url().optional(),
+    data_id: zod_1.z.string().or(zod_1.z.number()).optional(),
+    data_geo: zod_1.z.array(zod_1.z.string()).optional(),
+    data_text: zod_1.z.string().optional(),
+    data_url: zod_1.z.string().url().optional(),
+    data_original_type: zod_1.z.string().optional(),
+    data_language: zod_1.z.string().optional(),
+    data_sentiment: zod_1.z.string().optional(),
+    data_timestamp: zod_1.z.number().optional(),
     is_reply: zod_1.z.boolean().optional(),
     reply_to_message_id: zod_1.z.string().or(zod_1.z.number()).optional(),
     metadata: zod_1.z.record(zod_1.z.string(), zod_1.z.any()).optional(),
@@ -133,11 +134,15 @@ exports.zodDataSchema = zod_1.z.object({
     }))
         .optional(),
     author: zod_1.z.string().optional(),
-    replies: zod_1.z.any().optional(),
-    entities: zod_1.z.any().optional(),
     author_username: zod_1.z.string().optional(),
+    replies: zod_1.z.any().optional(),
     author_id: zod_1.z.string().optional(),
     group_id: zod_1.z.string().optional(), // e.g., 'default_pipeline'
+    // translated_text: z.string().optional(), // translated text if available - most of the time it will be translated to English
+    // entities: z.any().optional(), // array of reply texts or IDs
+    // source_public_id: z.string().or(z.number()), // e.g., 'telegram:1234567890' (message_id)
+    // platform_id: z.string().or(z.number()), // channel_id reference to Source source_id
+    // original_text_id: z.string().or(z.number()), // message_id
 });
 // Helper function to get allowed entities for a platform
 const getAllowedEntitiesForPlatform = (platform) => {

package/index.ts

@@ -11,27 +11,29 @@ import {
 
 export const MongoDataSchema = new Schema(
   {
-    source_id: { type: Schema.Types.Mixed, required: true }, // reference to Source _id
-    source_public_id: { type: Schema.Types.Mixed }, // e.g., 'telegram:1234567890' (message_id)
-    source_name: { type: String, required: true }, // e.g., 'Telegram Channel Name'
-    platform_id: { type: Schema.Types.Mixed }, // channel_id reference to Source source_id
+    timestamp: { type: Number }, // INDEX - date in milliseconds
     platform: {
       type: String,
       enum: platformsList,
-      required: true,
     }, // e.g., 'telegram', 'facebook'
-    original_text_id: { type: Schema.Types.Mixed }, // message_id
-    original_text: { type: String }, // original text content
-    translated_text: { type: String }, // translated text if available
-    timestamp: { type: Number, required: true }, // date in milliseconds
-    language: { type: String },
-    text_geo: { type: [String], default: [] }, // e.g., ["sinjil", "ramallah", "west bank"]
-    source_geo: { type: String }, // e.g., 'hebron'
+    source_region_id: { type: String }, // e.g., 'hebron'
+    source_region_title: { type: String }, // e.g., 'hebron'
+    source_id: { type: Schema.Types.Mixed, required: true }, // INDEX - reference to Source _id
+    source_title: { type: String }, // e.g., 'Telegram Channel Name'
+    source_url: { type: String }, // e.g., 'https://t.me/telegram_channel_name'
+    data_id: { type: Schema.Types.Mixed }, // INDEX - original text ID (e.g., message_id)
+    data_geo: { type: [String], default: [] }, // INDEX - e.g., ["sinjil", "ramallah", "west bank"]
+    data_text: { type: String }, // processed text content
+    data_url: { type: String }, // original text URL if available
+    data_original_type: { type: String }, // e.g., 'post', 'comment', 'reply', 'video', 'image'
+    data_language: { type: String }, // detected language of the text
+    data_sentiment: { type: String }, // sentiment analysis result
+    data_timestamp: { type: Number }, // original post timestamp if different from ingestion timestamp
     is_reply: { type: Boolean }, // true if this text is a reply to another text
     reply_to_message_id: { type: Schema.Types.Mixed },
     metadata: { type: Object }, // platform-specific fields
-    created_at: { type: Number, default: Date.now },
-    updated_at: { type: Number, default: Date.now },
+    created_at: { type: Number, default: Date.now, required: true },
+    updated_at: { type: Number, default: Date.now, required: true },
     media: {
       type: [
         {
@@ -42,12 +44,10 @@ export const MongoDataSchema = new Schema(
       ],
     }, // media attachments
     author: { type: String }, // e.g., author name or ID
-    replies: { type: Schema.Types.Mixed }, // array of reply texts or IDs
-    entities: { type: Schema.Types.Mixed }, // array of entities
     author_username: { type: String }, // e.g., author username
+    replies: { type: Schema.Types.Mixed }, // array of reply texts or IDs
     author_id: { type: String }, // e.g., author ID
-    pipeline_name: { type: String }, // e.g., 'default_pipeline'
-    source_region: { type: String }, // e.g., "New York", "California"
+    group_id: { type: String }, // e.g., source group ID
   },
   {
     versionKey: false,
@@ -119,7 +119,6 @@ export const MongoSourceSchema = new Schema(
       required: true,
     }, // e.g., 'channel', 'group', 'page', 'profile', 'hashtag'
     public_id: { type: Schema.Types.Mixed }, // e.g., '@telegram_channel_id'
-    name: { type: String, required: true }, // e.g., 'Telegram Channel Name'
     url: { type: String }, // e.g., 'https://t.me/telegram_channel_name'
     description: { type: String }, // e.g., 'A channel about news and updates'
     language: { type: String },
@@ -147,105 +146,6 @@ export const MongoSourceSchema = new Schema(
       default: undefined,
     }, // user who requested this source
     notes: { type: String }, // internal notes about the source
-
-  },
-  {
-    versionKey: false,
-    toJSON: { virtuals: true },
-    toObject: { virtuals: true },
-  }
-);
-
-export const MongoQuerySchema = new Schema(
-  {
-    title: { type: String }, // e.g., 'Search Query'
-    query: { type: String }, // search query includes geo information where it should be extracted with LLM
-    sources: { type: [String] }, // array of source _id strings
-    geos: { type: [String], required: true }, // array of geo strings
-    geos_ids: { type: [String], required: true }, // array of geo _id strings
-    time_range: {
-      type: Schema.Types.Mixed, // can be relative or absolute time range
-      required: true,
-    },
-    user_id: { type: String, required: true }, // user identifier
-    user_instructions: { type: String }, // optional user instructions for the query
-    user_time_zone: { type: String }, // user's timezone
-    timestamp: { type: Number, default: Date.now }, // creation date
-  },
-  {
-    versionKey: false,
-    toJSON: { virtuals: true },
-    toObject: { virtuals: true },
-  }
-);
-
-export const MongoUserSettingsSchema = new Schema(
-  {
-    sources: { type: [String] }, // array of source _id strings
-    geos: { type: [String], default: [] }, // array of geo strings
-    time_range: {
-      type: Schema.Types.Mixed,
-    },
-    user_id: { type: String, required: true },
-    thread_id: { type: String },
-    instructions: { type: String }, // optional user instructions for the query
-    has_jobs_access: { type: Boolean, default: false }, // true if the user has access to jobs
-    active_jobs_limit: { type: Number, default: 5 }, // maximum number of active jobs allowed
-    created_at: { type: Date, default: Date.now }, // creation date
-    updated_at: { type: Date, default: Date.now }, // last update date
-    status: {
-      type: String,
-      enum: generalStatusList,
-      default: 'active',
-    }, // status of the user
-  },
-  {
-    versionKey: false,
-    toJSON: { virtuals: true },
-    toObject: { virtuals: true },
-  }
-);
-
-export const MongoConversationSchema = new Schema(
-  {
-    user_id: { type: String, required: true }, // user identifier
-    title: { type: String }, // optional title for the conversation
-    thread_id: { type: String }, // thread ID for the conversation
-    status: {
-      type: String,
-      enum: generalStatusList,
-      default: 'active',
-    }, // status of the conversation
-    created_at: { type: Date, default: Date.now }, // creation date
-    updated_at: { type: Date, default: Date.now }, // last update date
-    is_job: { type: Boolean, default: false }, // true if this is a job conversation
-  },
-  {
-    versionKey: false,
-    toJSON: { virtuals: true },
-    toObject: { virtuals: true },
-  }
-);
-
-export const MongoJobSchema = new Schema(
-  {
-    user_id: { type: String, required: true }, // user identifier
-    job_name: { type: String, required: true }, // name of the automation job
-    job_description: { type: String }, // description of the job
-    status: {
-      type: String,
-      enum: generalStatusList,
-      default: 'active',
-    }, // status of the job
-    created_at: { type: Number, default: Date.now }, // creation date
-    updated_at: { type: Number, default: Date.now }, // last update date
-    schedule: { type: String }, // cron schedule for the job
-    schedule_text: { type: String }, // cron schedule in free-text format
-    active_until: { type: Number || undefined }, // optional date until which the job is active
-    query: { type: Schema.Types.Mixed, required: true }, // query associated with the job
-    conversation_id: { type: String }, // optional conversation ID for the job
-    thread_id: { type: String }, // thread ID for the job
-    time_zone: { type: String }, // optional timezone for the job
   },
   {
     versionKey: false,

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "harvester_sdk",
-  "version": "1.0.5",
+  "version": "1.0.7",
   "description": "SDK for interacting with the Harvester API",
   "main": "dist/index.js",
   "types": "dist/index.d.ts",

package/types.ts

@@ -67,7 +67,6 @@ export const zodSourceSchema = z.object({
   platform: z.enum(platformsList), // e.g., 'telegram', 'facebook' // INDEX
   entity: z.enum(entityTypesList), // e.g., 'channel', 'group', 'page', 'profile', 'hashtag'
   public_id: z.string().or(z.number()).optional(), // e.g., '@telegram_channel_id'
-  name: z.string(), // e.g., 'Telegram Channel Name'
   url: z.string().url().optional(), // e.g., 'https://t.me/telegram_channel_name'
   description: z.string().optional(), // e.g., 'A channel about news and updates'
   language: z.string().optional(),
@@ -114,20 +113,22 @@ export const zodGeoSelectionSchema = z.object({
 });
 
 export const zodDataSchema = z.object({
-  // Filterable fields
-  text_geo: z.array(z.string()), // INDEX - e.g., ["sinjil", "ramallah", "west bank"]
-  timestamp: z.number(), // INDEX - date in milliseconds - e.g., 1751210833000
-  platform: z.enum(platformsList), // e.g., 'telegram', 'facebook'
-  source_region: z.string().optional(), // e.g., 'hebron'
   _id: z.string().optional(),
+  timestamp: z.number().optional(), // INDEX - date in milliseconds - e.g., 1751210833000
+  platform: z.enum(platformsList).optional(), // e.g., 'telegram', 'facebook'
+  source_region_id: z.string().optional(), // e.g., 'hebron'
+  source_region_title: z.string().optional(), // e.g., 'hebron'
   source_id: z.string().or(z.number()), // INDEX - reference to Source _id (e.g., '60c72b2f9b1e8d3f4c8b4567')
-  source_public_id: z.string().or(z.number()), // e.g., 'telegram:1234567890' (message_id)
-  source_name: z.string(), // e.g., 'Telegram Channel Name'
-  platform_id: z.string().or(z.number()), // channel_id reference to Source source_id
-  original_text_id: z.string().or(z.number()), // message_id
-  original_text: z.string().optional(), // original text content
-  translated_text: z.string().optional(), // translated text if available - most of the time it will be translated to English
-  language: z.string().optional(),
+  source_title: z.string().optional(), // e.g., 'Telegram Channel Name'
+  source_url: z.string().url().optional(), // e.g., 'https://t.me/telegram_channel_name'
+  data_id: z.string().or(z.number()).optional(), // INDEX - original text ID (e.g., message_id)
+  data_geo: z.array(z.string()).optional(), // INDEX - e.g., ["sinjil", "ramallah", "west bank"]
+  data_text: z.string().optional(), // processed text content
+  data_url: z.string().url().optional(), // original text URL if available
+  data_original_type: z.string().optional(), // e.g., 'post', 'comment', 'reply', 'video', 'image'
+  data_language: z.string().optional(), // detected language of the text
+  data_sentiment: z.string().optional(), // sentiment analysis result
+  data_timestamp: z.number().optional(), // original post timestamp if different from ingestion timestamp
   is_reply: z.boolean().optional(), // true if this text is a reply to another text
   reply_to_message_id: z.string().or(z.number()).optional(),
   metadata: z.record(z.string(), z.any()).optional(), // platform-specific fields
@@ -143,20 +144,17 @@ export const zodDataSchema = z.object({
     )
     .optional(), // media attachments
   author: z.string().optional(), // e.g., author name or ID
-  replies: z.any().optional(), // array of reply texts or IDs
-  entities: z.any().optional(), // array of reply texts or IDs
   author_username: z.string().optional(), // e.g., author username
+  replies: z.any().optional(), // array of reply texts or IDs
   author_id: z.string().optional(), // e.g., author ID
   group_id: z.string().optional(), // e.g., 'default_pipeline'
+  // translated_text: z.string().optional(), // translated text if available - most of the time it will be translated to English
+  // entities: z.any().optional(), // array of reply texts or IDs
+  // source_public_id: z.string().or(z.number()), // e.g., 'telegram:1234567890' (message_id)
+  // platform_id: z.string().or(z.number()), // channel_id reference to Source source_id
+  // original_text_id: z.string().or(z.number()), // message_id
 });
 
-/**
- * 'approved' - active and approved sources,
- * 'back_to_business' - sources that were paused and now resumed,
- * 'pending' - sources that are pending approval,
- * 'inactive' - sources that are inactive,
- * 'in_review' - sources that are under review
- */
 export type RegionType = z.infer<typeof zodRegionSchema>;
 export type SourceGroupType = z.infer<typeof zodSourceGroupSchema>;
 export type SourceType = z.infer<typeof zodSourceSchema>;