ahok-skill 1.3.1

package/src/sources/google_drive.ts

@@ -0,0 +1,144 @@
+/**
+ * google drive source for openmemory - production grade
+ * requires: googleapis
+ * env vars: GOOGLE_SERVICE_ACCOUNT_FILE or GOOGLE_CREDENTIALS_JSON
+ */
+
+import { base_source, source_config_error, source_item, source_content } from './base';
+
+export class google_drive_source extends base_source {
+    name = 'google_drive';
+    private service: any = null;
+    private auth: any = null;
+
+    async _connect(creds: Record<string, any>): Promise<boolean> {
+        let google: any;
+        try {
+            google = await import('googleapis').then(m => m.google);
+        } catch {
+            throw new source_config_error('missing deps: npm install googleapis', this.name);
+        }
+
+        const scopes = ['https://www.googleapis.com/auth/drive.readonly'];
+
+        if (creds.credentials_json) {
+            this.auth = new google.auth.GoogleAuth({
+                credentials: creds.credentials_json,
+                scopes
+            });
+        } else if (creds.service_account_file) {
+            this.auth = new google.auth.GoogleAuth({
+                keyFile: creds.service_account_file,
+                scopes
+            });
+        } else if (process.env.GOOGLE_CREDENTIALS_JSON) {
+            this.auth = new google.auth.GoogleAuth({
+                credentials: JSON.parse(process.env.GOOGLE_CREDENTIALS_JSON),
+                scopes
+            });
+        } else if (process.env.GOOGLE_SERVICE_ACCOUNT_FILE) {
+            this.auth = new google.auth.GoogleAuth({
+                keyFile: process.env.GOOGLE_SERVICE_ACCOUNT_FILE,
+                scopes
+            });
+        } else {
+            throw new source_config_error(
+                'no credentials: set GOOGLE_SERVICE_ACCOUNT_FILE or GOOGLE_CREDENTIALS_JSON',
+                this.name
+            );
+        }
+
+        this.service = google.drive({ version: 'v3', auth: this.auth });
+        return true;
+    }
+
+    async _list_items(filters: Record<string, any>): Promise<source_item[]> {
+        const q_parts = ['trashed=false'];
+
+        if (filters.folder_id) {
+            q_parts.push(`'${filters.folder_id}' in parents`);
+        }
+
+        if (filters.mime_types?.length) {
+            const mime_q = filters.mime_types.map((m: string) => `mimeType='${m}'`).join(' or ');
+            q_parts.push(`(${mime_q})`);
+        }
+
+        const query = q_parts.join(' and ');
+        const results: source_item[] = [];
+        let page_token: string | undefined;
+
+        do {
+            const resp = await this.service.files.list({
+                q: query,
+                spaces: 'drive',
+                fields: 'nextPageToken, files(id, name, mimeType, modifiedTime, size)',
+                pageToken: page_token,
+                pageSize: 100
+            });
+
+            for (const f of resp.data.files || []) {
+                results.push({
+                    id: f.id!,
+                    name: f.name!,
+                    type: f.mimeType!,
+                    modified: f.modifiedTime,
+                    size: f.size
+                });
+            }
+
+            page_token = resp.data.nextPageToken;
+        } while (page_token);
+
+        return results;
+    }
+
+    async _fetch_item(item_id: string): Promise<source_content> {
+        const meta = await this.service.files.get({
+            fileId: item_id,
+            fields: 'id,name,mimeType'
+        });
+
+        const mime = meta.data.mimeType;
+        let text = '';
+        let data: string | Buffer = '';
+
+        // google docs -> export as text
+        if (mime === 'application/vnd.google-apps.document') {
+            const resp = await this.service.files.export({ fileId: item_id, mimeType: 'text/plain' });
+            text = resp.data;
+            data = text;
+        }
+        // google sheets -> export as csv
+        else if (mime === 'application/vnd.google-apps.spreadsheet') {
+            const resp = await this.service.files.export({ fileId: item_id, mimeType: 'text/csv' });
+            text = resp.data;
+            data = text;
+        }
+        // google slides -> export as plain text
+        else if (mime === 'application/vnd.google-apps.presentation') {
+            const resp = await this.service.files.export({ fileId: item_id, mimeType: 'text/plain' });
+            text = resp.data;
+            data = text;
+        }
+        // other files -> download raw
+        else {
+            const resp = await this.service.files.get({ fileId: item_id, alt: 'media' }, { responseType: 'arraybuffer' });
+            data = Buffer.from(resp.data);
+            try {
+                text = data.toString('utf-8');
+            } catch {
+                text = '';
+            }
+        }
+
+        return {
+            id: item_id,
+            name: meta.data.name!,
+            type: mime!,
+            text,
+            data,
+            meta: { source: 'google_drive', file_id: item_id, mime_type: mime }
+        };
+    }
+}

package/src/sources/google_sheets.ts

@@ -0,0 +1,85 @@
+/**
+ * google sheets source for openmemory - production grade
+ * requires: googleapis
+ * env vars: GOOGLE_SERVICE_ACCOUNT_FILE or GOOGLE_CREDENTIALS_JSON
+ */
+
+import { base_source, source_config_error, source_item, source_content } from './base';
+
+export class google_sheets_source extends base_source {
+    name = 'google_sheets';
+    private service: any = null;
+    private auth: any = null;
+
+    async _connect(creds: Record<string, any>): Promise<boolean> {
+        let google: any;
+        try {
+            google = await import('googleapis').then(m => m.google);
+        } catch {
+            throw new source_config_error('missing deps: npm install googleapis', this.name);
+        }
+
+        const scopes = ['https://www.googleapis.com/auth/spreadsheets.readonly'];
+
+        if (creds.credentials_json) {
+            this.auth = new google.auth.GoogleAuth({ credentials: creds.credentials_json, scopes });
+        } else if (creds.service_account_file) {
+            this.auth = new google.auth.GoogleAuth({ keyFile: creds.service_account_file, scopes });
+        } else if (process.env.GOOGLE_CREDENTIALS_JSON) {
+            this.auth = new google.auth.GoogleAuth({ credentials: JSON.parse(process.env.GOOGLE_CREDENTIALS_JSON), scopes });
+        } else if (process.env.GOOGLE_SERVICE_ACCOUNT_FILE) {
+            this.auth = new google.auth.GoogleAuth({ keyFile: process.env.GOOGLE_SERVICE_ACCOUNT_FILE, scopes });
+        } else {
+            throw new source_config_error('no credentials: set GOOGLE_SERVICE_ACCOUNT_FILE or GOOGLE_CREDENTIALS_JSON', this.name);
+        }
+
+        this.service = google.sheets({ version: 'v4', auth: this.auth });
+        return true;
+    }
+
+    async _list_items(filters: Record<string, any>): Promise<source_item[]> {
+        if (!filters.spreadsheet_id) {
+            throw new source_config_error('spreadsheet_id is required', this.name);
+        }
+
+        const meta = await this.service.spreadsheets.get({ spreadsheetId: filters.spreadsheet_id });
+
+        return (meta.data.sheets || []).map((sheet: any, i: number) => ({
+            id: `${filters.spreadsheet_id}!${sheet.properties?.title || 'Sheet1'}`,
+            name: sheet.properties?.title || 'Sheet1',
+            type: 'sheet',
+            index: i,
+            spreadsheet_id: filters.spreadsheet_id
+        }));
+    }
+
+    async _fetch_item(item_id: string): Promise<source_content> {
+        const [spreadsheet_id, sheet_range] = item_id.includes('!')
+            ? item_id.split('!', 2)
+            : [item_id, 'A:ZZ'];
+
+        const result = await this.service.spreadsheets.values.get({
+            spreadsheetId: spreadsheet_id,
+            range: sheet_range
+        });
+
+        const values = result.data.values || [];
+
+        // convert to markdown table
+        const lines = values.map((row: any[], i: number) => {
+            const line = row.map(String).join(' | ');
+            return i === 0 ? `${line}\n${row.map(() => '---').join(' | ')}` : line;
+        });
+
+        const text = lines.join('\n');
+
+        return {
+            id: item_id,
+            name: sheet_range,
+            type: 'spreadsheet',
+            text,
+            data: text,
+            meta: { source: 'google_sheets', spreadsheet_id, range: sheet_range, row_count: values.length }
+        };
+    }
+}

package/src/sources/google_slides.ts

@@ -0,0 +1,115 @@
+/**
+ * google slides source for openmemory - production grade
+ * requires: googleapis
+ * env vars: GOOGLE_SERVICE_ACCOUNT_FILE or GOOGLE_CREDENTIALS_JSON
+ */
+
+import { base_source, source_config_error, source_item, source_content } from './base';
+
+export class google_slides_source extends base_source {
+    name = 'google_slides';
+    private service: any = null;
+    private auth: any = null;
+
+    async _connect(creds: Record<string, any>): Promise<boolean> {
+        let google: any;
+        try {
+            google = await import('googleapis').then(m => m.google);
+        } catch {
+            throw new source_config_error('missing deps: npm install googleapis', this.name);
+        }
+
+        const scopes = ['https://www.googleapis.com/auth/presentations.readonly'];
+
+        if (creds.credentials_json) {
+            this.auth = new google.auth.GoogleAuth({ credentials: creds.credentials_json, scopes });
+        } else if (creds.service_account_file) {
+            this.auth = new google.auth.GoogleAuth({ keyFile: creds.service_account_file, scopes });
+        } else if (process.env.GOOGLE_CREDENTIALS_JSON) {
+            this.auth = new google.auth.GoogleAuth({ credentials: JSON.parse(process.env.GOOGLE_CREDENTIALS_JSON), scopes });
+        } else if (process.env.GOOGLE_SERVICE_ACCOUNT_FILE) {
+            this.auth = new google.auth.GoogleAuth({ keyFile: process.env.GOOGLE_SERVICE_ACCOUNT_FILE, scopes });
+        } else {
+            throw new source_config_error('no credentials: set GOOGLE_SERVICE_ACCOUNT_FILE or GOOGLE_CREDENTIALS_JSON', this.name);
+        }
+
+        this.service = google.slides({ version: 'v1', auth: this.auth });
+        return true;
+    }
+
+    async _list_items(filters: Record<string, any>): Promise<source_item[]> {
+        if (!filters.presentation_id) {
+            throw new source_config_error('presentation_id is required', this.name);
+        }
+
+        const pres = await this.service.presentations.get({ presentationId: filters.presentation_id });
+
+        return (pres.data.slides || []).map((slide: any, i: number) => ({
+            id: `${filters.presentation_id}#${slide.objectId}`,
+            name: `Slide ${i + 1}`,
+            type: 'slide',
+            index: i,
+            presentation_id: filters.presentation_id,
+            object_id: slide.objectId
+        }));
+    }
+
+    async _fetch_item(item_id: string): Promise<source_content> {
+        const [presentation_id, slide_id] = item_id.includes('#')
+            ? item_id.split('#', 2)
+            : [item_id, null];
+
+        const pres = await this.service.presentations.get({ presentationId: presentation_id });
+
+        const extract_text = (element: any): string => {
+            const texts: string[] = [];
+
+            if (element.shape?.text) {
+                for (const te of element.shape.text.textElements || []) {
+                    if (te.textRun) texts.push(te.textRun.content || '');
+                }
+            }
+
+            if (element.table) {
+                for (const row of element.table.tableRows || []) {
+                    for (const cell of row.tableCells || []) {
+                        if (cell.text) {
+                            for (const te of cell.text.textElements || []) {
+                                if (te.textRun) texts.push(te.textRun.content || '');
+                            }
+                        }
+                    }
+                }
+            }
+
+            return texts.join('');
+        };
+
+        const all_text: string[] = [];
+
+        for (let i = 0; i < (pres.data.slides || []).length; i++) {
+            const slide = pres.data.slides![i];
+            if (slide_id && slide.objectId !== slide_id) continue;
+
+            const slide_texts = [`## Slide ${i + 1}`];
+
+            for (const element of slide.pageElements || []) {
+                const txt = extract_text(element);
+                if (txt.trim()) slide_texts.push(txt.trim());
+            }
+
+            all_text.push(...slide_texts);
+        }
+
+        const text = all_text.join('\n\n');
+
+        return {
+            id: item_id,
+            name: pres.data.title || 'Untitled Presentation',
+            type: 'presentation',
+            text,
+            data: text,
+            meta: { source: 'google_slides', presentation_id, slide_count: pres.data.slides?.length || 0 }
+        };
+    }
+}

package/src/sources/index.ts

@@ -0,0 +1,19 @@
+/**
+ * openmemory sources - production-grade data source integrations
+ * 
+ * provides connectors for external data sources:
+ * - google drive, sheets, slides
+ * - notion
+ * - onedrive
+ * - github
+ * - web crawler
+ */
+
+export * from './base';
+export * from './google_drive';
+export * from './google_sheets';
+export * from './google_slides';
+export * from './notion';
+export * from './onedrive';
+export * from './github';
+export * from './web_crawler';

package/src/sources/notion.ts

@@ -0,0 +1,148 @@
+/**
+ * notion source for openmemory - production grade
+ * requires: @notionhq/client
+ * env vars: NOTION_API_KEY
+ */
+
+import { base_source, source_config_error, source_item, source_content } from './base';
+
+export class notion_source extends base_source {
+    name = 'notion';
+    private client: any = null;
+
+    async _connect(creds: Record<string, any>): Promise<boolean> {
+        let Client: any;
+        try {
+            Client = await import('@notionhq/client').then(m => m.Client);
+        } catch {
+            throw new source_config_error('missing deps: npm install @notionhq/client', this.name);
+        }
+
+        const api_key = creds.api_key || process.env.NOTION_API_KEY;
+
+        if (!api_key) {
+            throw new source_config_error('no credentials: set NOTION_API_KEY', this.name);
+        }
+
+        this.client = new Client({ auth: api_key });
+        return true;
+    }
+
+    private extract_title(page: any): string {
+        const props = page.properties || {};
+        for (const prop of Object.values(props) as any[]) {
+            if (prop.type === 'title' && prop.title?.[0]) {
+                return prop.title[0].plain_text || '';
+            }
+        }
+        return '';
+    }
+
+    async _list_items(filters: Record<string, any>): Promise<source_item[]> {
+        const results: source_item[] = [];
+
+        if (filters.database_id) {
+            let has_more = true;
+            let start_cursor: string | undefined;
+
+            while (has_more) {
+                const resp = await this.client.databases.query({
+                    database_id: filters.database_id,
+                    start_cursor
+                });
+
+                for (const page of resp.results) {
+                    results.push({
+                        id: page.id,
+                        name: this.extract_title(page) || 'Untitled',
+                        type: 'page',
+                        url: page.url || '',
+                        last_edited: page.last_edited_time
+                    });
+                }
+
+                has_more = resp.has_more;
+                start_cursor = resp.next_cursor;
+            }
+        } else {
+            const resp = await this.client.search({ filter: { property: 'object', value: 'page' } });
+
+            for (const page of resp.results) {
+                results.push({
+                    id: page.id,
+                    name: this.extract_title(page) || 'Untitled',
+                    type: 'page',
+                    url: page.url || '',
+                    last_edited: page.last_edited_time
+                });
+            }
+        }
+
+        return results;
+    }
+
+    private block_to_text(block: any): string {
+        const texts: string[] = [];
+        const type = block.type;
+
+        const text_blocks = ['paragraph', 'heading_1', 'heading_2', 'heading_3',
+            'bulleted_list_item', 'numbered_list_item', 'quote', 'callout'];
+
+        if (text_blocks.includes(type)) {
+            const rich_text = block[type]?.rich_text || [];
+            for (const rt of rich_text) {
+                texts.push(rt.plain_text || '');
+            }
+        } else if (type === 'code') {
+            const rich_text = block.code?.rich_text || [];
+            const lang = block.code?.language || '';
+            const code = rich_text.map((rt: any) => rt.plain_text || '').join('');
+            texts.push(`\`\`\`${lang}\n${code}\n\`\`\``);
+        } else if (type === 'to_do') {
+            const checked = block.to_do?.checked || false;
+            const rich_text = block.to_do?.rich_text || [];
+            const prefix = checked ? '[x] ' : '[ ] ';
+            texts.push(prefix + rich_text.map((rt: any) => rt.plain_text || '').join(''));
+        }
+
+        return texts.join('');
+    }
+
+    async _fetch_item(item_id: string): Promise<source_content> {
+        const page = await this.client.pages.retrieve({ page_id: item_id });
+        const title = this.extract_title(page);
+
+        // get all blocks
+        const blocks: any[] = [];
+        let has_more = true;
+        let start_cursor: string | undefined;
+
+        while (has_more) {
+            const resp = await this.client.blocks.children.list({
+                block_id: item_id,
+                start_cursor
+            });
+            blocks.push(...resp.results);
+            has_more = resp.has_more;
+            start_cursor = resp.next_cursor;
+        }
+
+        const text_parts = title ? [`# ${title}`] : [];
+
+        for (const block of blocks) {
+            const txt = this.block_to_text(block);
+            if (txt.trim()) text_parts.push(txt);
+        }
+
+        const text = text_parts.join('\n\n');
+
+        return {
+            id: item_id,
+            name: title || 'Untitled',
+            type: 'notion_page',
+            text,
+            data: text,
+            meta: { source: 'notion', page_id: item_id, url: page.url || '', block_count: blocks.length }
+        };
+    }
+}

package/src/sources/onedrive.ts

@@ -0,0 +1,131 @@
+/**
+ * onedrive source for openmemory - production grade
+ * requires: @azure/msal-node
+ * env vars: AZURE_CLIENT_ID, AZURE_CLIENT_SECRET, AZURE_TENANT_ID
+ */
+
+import { base_source, source_config_error, source_auth_error, source_item, source_content } from './base';
+
+export class onedrive_source extends base_source {
+    name = 'onedrive';
+    private access_token: string | null = null;
+    private graph_url = 'https://graph.microsoft.com/v1.0';
+
+    async _connect(creds: Record<string, any>): Promise<boolean> {
+        if (creds.access_token) {
+            this.access_token = creds.access_token;
+            return true;
+        }
+
+        let msal: any;
+        try {
+            msal = await import('@azure/msal-node');
+        } catch {
+            throw new source_config_error('missing deps: npm install @azure/msal-node', this.name);
+        }
+
+        const client_id = creds.client_id || process.env.AZURE_CLIENT_ID;
+        const client_secret = creds.client_secret || process.env.AZURE_CLIENT_SECRET;
+        const tenant_id = creds.tenant_id || process.env.AZURE_TENANT_ID;
+
+        if (!client_id || !client_secret || !tenant_id) {
+            throw new source_config_error(
+                'no credentials: set AZURE_CLIENT_ID, AZURE_CLIENT_SECRET, AZURE_TENANT_ID',
+                this.name
+            );
+        }
+
+        const app = new msal.ConfidentialClientApplication({
+            auth: {
+                clientId: client_id,
+                clientSecret: client_secret,
+                authority: `https://login.microsoftonline.com/${tenant_id}`
+            }
+        });
+
+        const result = await app.acquireTokenByClientCredential({
+            scopes: ['https://graph.microsoft.com/.default']
+        });
+
+        if (result?.accessToken) {
+            this.access_token = result.accessToken;
+            return true;
+        }
+
+        throw new source_auth_error('auth failed: no access token returned', this.name);
+    }
+
+    async _list_items(filters: Record<string, any>): Promise<source_item[]> {
+        const folder_path = filters.folder_path || '/';
+        const user_principal = filters.user_principal;
+
+        const base = user_principal
+            ? `${this.graph_url}/users/${user_principal}/drive`
+            : `${this.graph_url}/me/drive`;
+
+        const url = folder_path === '/'
+            ? `${base}/root/children`
+            : `${base}/root:/${folder_path.replace(/^\/|\/$/g, '')}:/children`;
+
+        const results: source_item[] = [];
+        let next_url: string | null = url;
+
+        while (next_url) {
+            const resp: Response = await fetch(next_url, {
+                headers: { Authorization: `Bearer ${this.access_token}` }
+            });
+
+            if (!resp.ok) throw new Error(`http ${resp.status}: ${resp.statusText}`);
+
+            const data: any = await resp.json();
+
+            for (const item of data.value || []) {
+                results.push({
+                    id: item.id,
+                    name: item.name,
+                    type: 'folder' in item ? 'folder' : item.file?.mimeType || 'file',
+                    size: item.size || 0,
+                    modified: item.lastModifiedDateTime,
+                    path: item.parentReference?.path || ''
+                });
+            }
+
+            next_url = data['@odata.nextLink'] || null;
+        }
+
+        return results;
+    }
+
+    async _fetch_item(item_id: string): Promise<source_content> {
+        const base = `${this.graph_url}/me/drive`;
+
+        const meta_resp = await fetch(`${base}/items/${item_id}`, {
+            headers: { Authorization: `Bearer ${this.access_token}` }
+        });
+
+        if (!meta_resp.ok) throw new Error(`http ${meta_resp.status}`);
+        const meta = await meta_resp.json();
+
+        const content_resp = await fetch(`${base}/items/${item_id}/content`, {
+            headers: { Authorization: `Bearer ${this.access_token}` },
+            redirect: 'follow'
+        });
+
+        if (!content_resp.ok) throw new Error(`http ${content_resp.status}`);
+        const data = Buffer.from(await content_resp.arrayBuffer());
+
+        let text = '';
+        try {
+            text = data.toString('utf-8');
+        } catch { }
+
+        return {
+            id: item_id,
+            name: meta.name || 'unknown',
+            type: meta.file?.mimeType || 'unknown',
+            text,
+            data,
+            meta: { source: 'onedrive', item_id, size: meta.size || 0, mime_type: meta.file?.mimeType || '' }
+        };
+    }
+}