RubyGems - multiwoven-integrations - Versions diffs - 0.32.2 → 0.33.0 - Mend

multiwoven-integrations 0.32.2 → 0.33.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.

Files changed (11) hide show

checksums.yaml CHANGED Viewed

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 51bfe2ae51ee59ed9206443a355de5a16ca11e28c3bb2fe734b1e47f9de0453c
-  data.tar.gz: 0b594fce51a1496dd0a08d8df98bb509ca234873a238479a9a72565c7d212f72
+  metadata.gz: 6be57be6129595ce97d90e8ae2458d9acf46f263d6385e19ab8d5e54f89bcc3a
+  data.tar.gz: 6054e513e6cd67226c5f35e9ab5f85dfe39ad3a13cab12ed2e1f6f33c08b68be
 SHA512:
-  metadata.gz: 5f70ba477eeaa43df25c71387dfa77b782e6911978a1508bda96e5524460f57ae0c020977fb6156e49294bb05e577b3b4d5713a2a02822d526b398dbbce69ca5
-  data.tar.gz: 53d6ab4863f6b2abbcd21790e4f139bfcbe62adeeeffd66afef97db91688cb3c8e02df5c19d0122d88cc3cf0cf029f335c5067f7a0a3fc81b7771b864471cde8
+  metadata.gz: aa601444cd95bd8a8b6404ff5c352c4d44a45a5a4d50af0a28a3bcb0a2a90163e1ee84ffb71939e575086821a54d88159fef1cbec260c7037ff423132b0840e7
+  data.tar.gz: b70732578f3dc563519db07e7ecb10df0022281e781099820a90f585791ca786c8eb23523fa93d2417acb1a51657e0f021b326822b5ca9845569ba57c288bef4

data/lib/multiwoven/integrations/core/constants.rb CHANGED Viewed

@@ -98,6 +98,22 @@ module Multiwoven
       FIRECRAWL_CRAWL_ACTIVE_URL = "https://api.firecrawl.dev/v1/crawl/active"
       FIRECRAWL_GET_CRAWL_URL = "https://api.firecrawl.dev/v1/crawl/%<id>s"
       FIRECRAWL_REQUEST_RATE_LIMIT = 5
+      # Amazon Textract (analyze_expense) fields
+      TEXTRACT_SUMMARY_FIELDS = {
+        "invoice_number" => "INVOICE_RECEIPT_ID",
+        "invoice_date" => "INVOICE_RECEIPT_DATE",
+        "purchase_order" => "PO_NUMBER",
+        "invoice_total" => "TOTAL",
+        "vendor_name" => "VENDOR_NAME"
+      }.freeze
+      TEXTRACT_LINE_ITEMS_FIELDS = {
+        "item_number" => "PRODUCT_CODE",
+        "item_description" => "ITEM",
+        "item_quantity" => "QUANTITY",
+        "item_price" => "UNIT_PRICE",
+        "line_total" => "PRICE"
+      }.freeze
     end
   end
 end

data/lib/multiwoven/integrations/destination/pinecone_db/client.rb CHANGED Viewed

@@ -104,15 +104,15 @@ module Multiwoven::Integrations::Destination
       end
       def parse_meta_data(vector_meta_data)
-        return {} if vector_meta_data.nil?
+        return {} unless vector_meta_data.is_a?(String)
-        metadata = vector_meta_data.to_s
-        metadata = metadata.gsub(/([{,]\s*)([A-Za-z_]\w*)(\s*:)/, '\1"\2"\3')
-        metadata = metadata.gsub(/:\s*([A-Za-z_]\w*)/, ': "\1"')
+        metadata = {}
+        # Use regex to parse the metadata string into a hash
+        vector_meta_data.scan(/([A-Za-z]+):\s*(.*?)(?=(?:\n?[A-Za-z]+:)|\z)/m).each do |key, value|
+          metadata[key.strip] = value.strip
+        end
-        JSON.parse(metadata)
-      rescue JSON::ParserError
-        {}
+        metadata
       end
       def send_to_pinecone(pinecone_index, record)

data/lib/multiwoven/integrations/rollout.rb CHANGED Viewed

@@ -2,7 +2,7 @@
 module Multiwoven
   module Integrations
-    VERSION = "0.32.2"
+    VERSION = "0.33.0"
     ENABLED_SOURCES = %w[
       Snowflake
@@ -32,6 +32,7 @@ module Multiwoven
       Qdrant
       Firecrawl
       Odoo
+      GoogleDrive
     ].freeze
     ENABLED_DESTINATIONS = %w[

data/lib/multiwoven/integrations/source/google_drive/client.rb ADDED Viewed

@@ -0,0 +1,215 @@
+# frozen_string_literal: true
+module Multiwoven::Integrations::Source
+  module GoogleDrive
+    include Multiwoven::Integrations::Core
+    FIELDS = "files(id, name, parents, mimeType), nextPageToken"
+    MAX_PER_PAGE = 1000
+    MIMETYPE_GOOGLE_DRIVE_FOLDER = "mimeType = 'application/vnd.google-apps.folder'"
+    class Client < SourceConnector
+      def check_connection(connection_config)
+        connection_config = connection_config.with_indifferent_access
+        client = create_connection(connection_config)
+        build_query(client)
+        success_status
+      rescue StandardError => e
+        failure_status(e)
+      end
+      def discover(_connection_config)
+        catalog_json = read_json(CATALOG_SPEC_PATH)
+        catalog = build_catalog(catalog_json)
+        catalog.to_multiwoven_message
+      rescue StandardError => e
+        handle_exception(e, {
+                           context: "GOOGLE_DRIVE:DISCOVER:EXCEPTION",
+                           type: "error"
+                         })
+      end
+      def read(sync_config)
+        connection_config = sync_config.source.connection_specification.with_indifferent_access
+        client = create_connection(connection_config)
+        query = sync_config.model.query
+        query = batched_query(query, sync_config.limit, sync_config.offset) unless sync_config.limit.nil? && sync_config.offset.nil?
+        records = query(client, query)
+        analyze_expenses(client, records)
+      rescue StandardError => e
+        handle_exception(e, {
+                           context: "GOOGLE_DRIVE:READ:EXCEPTION",
+                           type: "error",
+                           sync_id: sync_config.sync_id,
+                           sync_run_id: sync_config.sync_run_id
+                         })
+      end
+      private
+      def query(client, query)
+        limit = 0
+        offset = 0
+        query = query.gsub("\n", " ").gsub(/\s+/, " ")
+        limit = query.match(/LIMIT (\d+)/)[1].to_i if query.include? "LIMIT"
+        offset = query.match(/OFFSET (\d+)/)[1].to_i if query.include? "OFFSET"
+        query = query.match(/\((.*)\) AS/)[1] if query.include? "AS subquery"
+        columns = select_columns(query)
+        google_drive_query = build_query(client)
+        files = get_files(client, google_drive_query, limit, offset)
+        files.map do |file|
+          RecordMessage.new(data: prepare_invoice(file, columns), emitted_at: Time.now.to_i).to_multiwoven_message
+        end
+      end
+      # Reads files from Google Drive and sends them to Amazon Textract for analysis
+      def analyze_expenses(client, records)
+        textract = create_aws_connection
+        results = []
+        records.each do |record|
+          invoice = record.record.data
+          begin
+            byte_stream = StringIO.new
+            client.get_file(invoice["id"], download_dest: byte_stream)
+            byte_stream.rewind
+            analysis = textract.analyze_expense(document: { bytes: byte_stream.read })
+            invoice = extract_invoice_data(invoice, analysis)
+          rescue Aws::Textract::Errors::UnsupportedDocumentException => e
+            invoice[:exception] = "Document format not supported." if invoice.key?(:exception)
+            handle_exception(e, {
+                               context: "GOOGLE_DRIVE:READ:EXTRACT:EXCEPTION",
+                               type: "error"
+                             })
+          rescue StandardError => e
+            invoice[:exception] = e.message if invoice.key?(:exception)
+            handle_exception(e, {
+                               context: "GOOGLE_DRIVE:READ:EXTRACT:EXCEPTION",
+                               type: "error"
+                             })
+          end
+          results.append(RecordMessage.new(data: invoice, emitted_at: Time.now.to_i).to_multiwoven_message)
+        end
+        results
+      end
+      def build_query(client)
+        query = "mimeType != 'application/vnd.google-apps.folder'"
+        if @options[:folder]
+          folder_query = "#{MIMETYPE_GOOGLE_DRIVE_FOLDER} and (name = '#{@options[:folder]}')"
+          response = client.list_files(include_items_from_all_drives: true, supports_all_drives: true, q: folder_query, fields: FIELDS)
+          raise "Specified folder does not exist" if response.files.empty?
+          parent_id = response.files.first.id
+          parents_query = "'#{parent_id}' in parents"
+        end
+        if @options[:subfolders]
+          subfolders_query = MIMETYPE_GOOGLE_DRIVE_FOLDER
+          subfolders_query += "and #{parents_query}" if parents_query
+          response = client.list_files(include_items_from_all_drives: true, supports_all_drives: true, q: subfolders_query, fields: FIELDS)
+          subfolders_ids = response.files.map { |file| "'#{file.id}'" }
+          parents_query = "(#{subfolders_ids.join(" in parents or ")} in parents)"
+        end
+        query += " and mimeType = '#{@options[:file_type]}'" if @options[:file_type]
+        query += " and #{parents_query}" if parents_query
+        query
+      end
+      def get_files(client, query, limit, offset)
+        total_fetched = 0
+        result = []
+        return result if offset.positive? && !@next_page_token
+        while total_fetched < limit
+          batch_limit = [MAX_PER_PAGE, limit - total_fetched].min
+          response = if @next_page_token
+                       client.list_files(include_items_from_all_drives: true, supports_all_drives: true, q: query, fields: FIELDS, page_size: batch_limit, page_token: @next_page_token)
+                     else
+                       client.list_files(include_items_from_all_drives: true, supports_all_drives: true, q: query, fields: FIELDS, page_size: batch_limit)
+                     end
+          break if response.files.empty?
+          result.push(*response.files)
+          @next_page_token = response.next_page_token
+          break unless response.next_page_token
+          total_fetched += response.files.size
+        end
+        result
+      end
+      def select_columns(query)
+        columns = query.match(/SELECT (.*) FROM/)[1]
+        all_columns = %w[line_items id file_name] + TEXTRACT_SUMMARY_FIELDS.keys
+        @options[:fields] = all_columns if @options[:fields].empty?
+        return @options[:fields] if columns.include?("*")
+        columns = columns.split(",").map(&:strip)
+        raise "Column(s) #{(columns - all_columns).join(", ")} not valid." if (columns - all_columns).length.positive?
+        columns & all_columns
+      end
+      def prepare_invoice(file, columns)
+        invoice = {}
+        columns.each { |column| invoice[column] = "" if TEXTRACT_SUMMARY_FIELDS.key?(column) }
+        invoice["line_items"] = [] if columns.any?("line_items")
+        invoice["id"] = file.id if columns.any?("id")
+        invoice["file_name"] = file.id if columns.any?("file_name")
+        invoice["exception"] = "" if columns.any?("exception")
+        invoice
+      end
+      def create_connection(connection_config)
+        @options = connection_config[:options]
+        credentials = connection_config[:credentials_json]
+        client = Google::Apis::DriveV3::DriveService.new
+        client.authorization = Google::Auth::ServiceAccountCredentials.make_creds(
+          json_key_io: StringIO.new(credentials.to_json),
+          scope: GOOGLE_SHEETS_SCOPE
+        )
+        client
+      end
+      def create_aws_connection
+        region = ENV["AWS_REGION"]
+        access_key_id = ENV["AWS_ACCESS_KEY_ID"]
+        secret_access_key = ENV["AWS_SECRET_ACCESS_KEY"]
+        credentials = Aws::Credentials.new(access_key_id, secret_access_key)
+        Aws::Textract::Client.new(region: region, credentials: credentials)
+      end
+      def extract_invoice_data(invoice, results)
+        expense_document = results.expense_documents[0]
+        (invoice.keys & TEXTRACT_SUMMARY_FIELDS.keys).each do |key|
+          invoice[key] = extract_field_value(expense_document.summary_fields, TEXTRACT_SUMMARY_FIELDS[key])
+        end
+        if invoice.key?("line_items")
+          expense_document.line_item_groups.each do |line_item_group|
+            line_item_group.line_items.each do |line_item|
+              extracted_line_item = {}
+              TEXTRACT_LINE_ITEMS_FIELDS.each_key do |key|
+                extracted_line_item[key] = extract_field_value(line_item.line_item_expense_fields, TEXTRACT_LINE_ITEMS_FIELDS[key])
+              end
+              invoice["line_items"] << extracted_line_item
+            end
+          end
+        end
+        invoice["line_items"] = invoice["line_items"].to_json
+        invoice.transform_keys(&:to_sym)
+      end
+      def extract_field_value(fields, selector)
+        selected_field = fields.select { |field| field.type.text == selector }.first
+        selected_field ? selected_field.value_detection.text : ""
+      end
+    end
+  end
+end

data/lib/multiwoven/integrations/source/google_drive/config/catalog.json ADDED Viewed

@@ -0,0 +1,63 @@
+{
+  "request_rate_limit": 6000,
+  "request_rate_limit_unit": "minute",
+  "request_rate_concurrency": 10,
+  "streams": [
+    {
+      "name": "invoices",
+      "action": "fetch",
+      "json_schema": {
+        "type": "object",
+        "properties": {
+          "id": {
+            "type": "string"
+          },
+          "vendor_name": {
+            "type": "string"
+          },
+          "file_name": {
+            "type": "string"
+          },
+          "exception": {
+            "type": "string"
+          },
+          "invoice_number": {
+            "type": "string"
+          },
+          "purchase_order": {
+            "type": "string"
+          },
+          "invoice_date": {
+            "type": "string"
+          },
+          "invoice_total": {
+            "type": "string"
+          },
+          "line_items": {
+            "type": "array",
+            "items": {
+              "type": "object",
+              "properties": {
+                "item_number": {
+                  "type": "string"
+                },
+                "item_description": {
+                  "type": "string"
+                },
+                "item_quantity": {
+                  "type": "string"
+                },
+                "item_price": {
+                  "type": "string"
+                },
+                "line_total": {
+                  "type": "string"
+                }
+              }
+            }
+          }
+        }
+      }
+    }
+  ]
+}

data/lib/multiwoven/integrations/source/google_drive/config/meta.json ADDED Viewed

@@ -0,0 +1,15 @@
+{
+  "data": {
+    "name": "GoogleDrive",
+    "title": "Google Drive",
+    "connector_type": "source",
+    "category": "File Storage",
+    "documentation_url": "https://docs.squared.ai/guides/sources/data-sources/google_drive",
+    "github_issue_label": "source-google-drive",
+    "icon": "icon.svg",
+    "license": "MIT",
+    "release_stage": "alpha",
+    "support_level": "community",
+    "tags": ["language:ruby", "multiwoven"]
+  }
+}

data/lib/multiwoven/integrations/source/google_drive/config/spec.json ADDED Viewed

@@ -0,0 +1,120 @@
+{
+  "documentation_url": "https://docs.squared.ai/guides/sources/data-sources/google_drive",
+  "stream_type": "dynamic",
+  "connector_query_type": "raw_sql",
+  "connection_specification": {
+    "$schema": "http://json-schema.org/draft-07/schema#",
+    "title": "GoogleDrive",
+    "type": "object",
+    "required": ["credentials_json"],
+    "properties": {
+      "credentials_json": {
+        "type": "object",
+        "description": "You can get the keys from the Google Cloud web console. First, go to the IAM page and select Service Accounts from the left menu. Next, locate your service account in the list, click on its Keys tab, and then click Add Key. Lastly, click Create new key and select JSON.",
+        "title": "",
+        "properties": {
+          "type": {
+            "type": "string",
+            "enum": ["service_account"]
+          },
+          "project_id": {
+            "type": "string"
+          },
+          "private_key_id": {
+            "type": "string",
+            "multiwoven_secret": true
+          },
+          "private_key": {
+            "type": "string",
+            "multiwoven_secret": true
+          },
+          "client_email": {
+            "type": "string",
+            "format": "email"
+          },
+          "client_id": {
+            "type": "string",
+            "multiwoven_secret": true
+          },
+          "auth_uri": {
+            "type": "string"
+          },
+          "token_uri": {
+            "type": "string"
+          },
+          "auth_provider_x509_cert_url": {
+            "type": "string"
+          },
+          "client_x509_cert_url": {
+            "type": "string",
+            "format": "uri"
+          },
+          "universe_domain": {
+            "type": "string"
+          }
+        },
+        "required": [
+          "type",
+          "project_id",
+          "private_key_id",
+          "private_key",
+          "client_email",
+          "client_id",
+          "auth_uri",
+          "token_uri",
+          "auth_provider_x509_cert_url",
+          "client_x509_cert_url",
+          "universe_domain"
+        ]
+      },
+      "options": {
+        "type": "object",
+        "description": "When the subfolders option is set to true, files will be read from the subfolders of the root or specified folder.",
+        "title": "Options",
+        "required": ["document_type"],
+        "properties": {
+          "folder": {
+            "type": "string",
+            "description": "When specified, reads files contained in the folder.",
+            "title": "Folder"
+          },
+          "subfolders": {
+            "type": "boolean",
+            "default": false,
+            "title": "Read from subfolders"
+          },
+          "file_type": {
+            "description": "The type of file to read",
+            "type": "string",
+            "title": "File Type",
+            "enum": [
+              "application/pdf"
+            ]
+          },
+          "document_type": {
+            "type": "string",
+            "enum": [ "invoices" ]
+          },
+          "fields": {
+            "type": "array",
+            "description": "Leave blank to extract all fields.",
+            "items": {
+              "type": "string",
+              "anyOf": [
+                { "const": "id", "title": "File Id" },
+                { "const": "file_name", "title": "File Name" },
+                { "const": "exception", "title": "Parsing Exception" },
+                { "const": "invoice_number", "title": "Invoice/Receipt Number" },
+                { "const": "invoice_date", "title": "Invoice/Receipt Date" },
+                { "const": "invoice_total", "title": "Invoice/Receipt Total" },
+                { "const": "purchase_order", "title": "Purchase Order Number" },
+                { "const": "line_items", "title": "Invoice Line Items (Code, Description, Quantity, Unit Price, Total Price)"}
+              ]
+            },
+            "uniqueItems": true
+          }
+        }
+      }
+    }
+  }
+}

data/lib/multiwoven/integrations/source/google_drive/icon.svg ADDED Viewed

@@ -0,0 +1,8 @@
+<svg viewBox="0 0 87.3 78" xmlns="http://www.w3.org/2000/svg">
+	<path d="m6.6 66.85 3.85 6.65c.8 1.4 1.95 2.5 3.3 3.3l13.75-23.8h-27.5c0 1.55.4 3.1 1.2 4.5z" fill="#0066da"/>
+	<path d="m43.65 25-13.75-23.8c-1.35.8-2.5 1.9-3.3 3.3l-25.4 44a9.06 9.06 0 0 0 -1.2 4.5h27.5z" fill="#00ac47"/>
+	<path d="m73.55 76.8c1.35-.8 2.5-1.9 3.3-3.3l1.6-2.75 7.65-13.25c.8-1.4 1.2-2.95 1.2-4.5h-27.502l5.852 11.5z" fill="#ea4335"/>
+	<path d="m43.65 25 13.75-23.8c-1.35-.8-2.9-1.2-4.5-1.2h-18.5c-1.6 0-3.15.45-4.5 1.2z" fill="#00832d"/>
+	<path d="m59.8 53h-32.3l-13.75 23.8c1.35.8 2.9 1.2 4.5 1.2h50.8c1.6 0 3.15-.45 4.5-1.2z" fill="#2684fc"/>
+	<path d="m73.4 26.5-12.7-22c-.8-1.4-1.95-2.5-3.3-3.3l-13.75 23.8 16.15 28h27.45c0-1.55-.4-3.1-1.2-4.5z" fill="#ffba00"/>
+</svg>

data/lib/multiwoven/integrations.rb CHANGED Viewed

@@ -42,6 +42,9 @@ require "pinecone"
 require "intuit-oauth"
 require "nokogiri"
 require "xmlrpc/client"
+require "googleauth"
+require "google/apis/drive_v3"
+require "aws-sdk-textract"
 # Service
 require_relative "integrations/config"
@@ -92,6 +95,7 @@ require_relative "integrations/source/pinecone_db/client"
 require_relative "integrations/source/qdrant/client"
 require_relative "integrations/source/firecrawl/client"
 require_relative "integrations/source/odoo/client"
+require_relative "integrations/source/google_drive/client"
 # Destination
 require_relative "integrations/destination/klaviyo/client"

metadata CHANGED Viewed

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: multiwoven-integrations
 version: !ruby/object:Gem::Version
-  version: 0.32.2
+  version: 0.33.0
 platform: ruby
 authors:
 - Subin T P
 autorequire:
 bindir: exe
 cert_chain: []
-date: 2025-07-21 00:00:00.000000000 Z
+date: 2025-07-31 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: activesupport
@@ -765,6 +765,11 @@ files:
 - lib/multiwoven/integrations/source/generic_open_ai/config/meta.json
 - lib/multiwoven/integrations/source/generic_open_ai/config/spec.json
 - lib/multiwoven/integrations/source/generic_open_ai/icon.svg
+- lib/multiwoven/integrations/source/google_drive/client.rb
+- lib/multiwoven/integrations/source/google_drive/config/catalog.json
+- lib/multiwoven/integrations/source/google_drive/config/meta.json
+- lib/multiwoven/integrations/source/google_drive/config/spec.json
+- lib/multiwoven/integrations/source/google_drive/icon.svg
 - lib/multiwoven/integrations/source/google_vertex_model/client.rb
 - lib/multiwoven/integrations/source/google_vertex_model/config/catalog.json
 - lib/multiwoven/integrations/source/google_vertex_model/config/meta.json