twingly-search 5.0.1 → 5.1.0

data/lib/twingly/search/parser.rb

@@ -6,16 +6,16 @@ module Twingly
       # Parse an API response body.
       #
       # @param [String] document containing an API response XML.
-      # @raise [Error] which error depends on the API response (see {Error.from_api_response_message}).
+      # @raise [Error] which error depends on the API response (see {Error.from_api_response}).
       # @return [Result] containing the result.
       def parse(document)
         nokogiri = Nokogiri::XML(document)
 
-        failure = nokogiri.at_xpath('//name:blogstream/name:operationResult[@resultType="failure"]', name: 'http://www.twingly.com')
+        failure = nokogiri.at_xpath('/error')
         handle_failure(failure) if failure
 
         data_node = nokogiri.at_xpath('/twinglydata')
-        handle_non_xml_document(nokogiri) unless data_node
+        handle_non_xml_document(document) unless data_node
 
         create_result(data_node)
       end
@@ -27,41 +27,67 @@ module Twingly
         result.number_of_matches_returned = data_node.attribute('numberOfMatchesReturned').value.to_i
         result.number_of_matches_total    = data_node.attribute('numberOfMatchesTotal').value.to_i
         result.seconds_elapsed            = data_node.attribute('secondsElapsed').value.to_f
+        result.incomplete_result          = incomplete_result?(data_node)
 
-        data_node.xpath('//post[@contentType="blog"]').each do |post|
+        data_node.xpath('//post').each do |post|
           result.posts << parse_post(post)
         end
 
         result
       end
 
+      def incomplete_result?(data_node)
+        data_node.attribute('incompleteResult').value == "true"
+      end
+
       def parse_post(element)
         post_params = {}
         element.element_children.each do |child|
-          if child.name == 'tags'
-            post_params[child.name] = parse_tags(child)
-          else
-            post_params[child.name] = child.text
-          end
+          post_params[child.name] =
+            case child.name
+            when *%w(tags links images)
+              parse_array(child)
+            when "coordinates"
+              parse_coordinates(child)
+            else
+              child.text
+            end
         end
+
         post = Post.new
         post.set_values(post_params)
         post
       end
 
-      def parse_tags(element)
+      def parse_array(element)
         element.element_children.map do |child|
           child.text
         end
       end
 
+      # TODO: Decide if a class or hash should be used...
+      def parse_coordinates(element)
+        return {} if element.children.empty?
+
+        {
+          latitude:  element.at_xpath("latitude/text()"),
+          longitude: element.at_xpath("longitude/text()"),
+        }
+      end
+
       def handle_failure(failure)
-        fail Error.from_api_response_message(failure.text)
+        code    = failure.attribute('code').value
+        message = failure.at_xpath('message').text
+
+        fail Error.from_api_response(code, message)
       end
 
       def handle_non_xml_document(document)
-        response_text = document.search('//text()').map(&:text)
-        fail ServerError, response_text
+        fail ServerError, "Failed to parse response: \"#{document}\""
+      end
+
+      def parse_time(time)
+        Time.parse(time)
       end
     end
   end

data/lib/twingly/search/post.rb

@@ -1,44 +1,88 @@
 # encoding: utf-8
 
-require 'date'
-
 module Twingly
   module Search
     # A blog post
     #
-    # @attr_reader [String] url the post URL.
-    # @attr_reader [String] title the post title.
-    # @attr_reader [String] summary the blog post text.
+    # @attr_reader [String] id the post ID (Twingly internal identification)
+    # @attr_reader [String] author the author of the blog post
+    # @attr_reader [String] url the post URL
+    # @attr_reader [String] title the post title
+    # @attr_reader [String] text the blog post text
     # @attr_reader [String] language_code ISO two letter language code for the
-    #   language that the post was written in.
-    # @attr_reader [Time] indexed the time, in UTC, when the post was indexed by Twingly.
-    # @attr_reader [Time] published the time, in UTC, when the post was published.
-    # @attr_reader [String] blog_url the blog URL.
-    # @attr_reader [String] blog_name name of the blog.
-    # @attr_reader [String] authority the blog's authority/influence.
-    #   See https://developer.twingly.com/resources/ranking/#authority
+    #   language that the post was written in
+    # @attr_reader [String] location_code ISO two letter country code for the
+    #   location of the blog
+    # @attr_reader [Hash] coordinates a hash containing :latitude and :longitude
+    #   from the post RSS
+    # @attr_reader [Array] links all links from the blog post to other resources
+    # @attr_reader [Array] tags the post tags/categories
+    # @attr_reader [Array] images image URLs from the post (currently not populated)
+    # @attr_reader [Time] indexed_at the time, in UTC, when the post was indexed by Twingly
+    # @attr_reader [Time] published_at the time, in UTC, when the post was published
+    # @attr_reader [Time] reindexed_at timestamp when the post last was changed in our database/index
+    # @attr_reader [String] inlinks_count number of links to this post that was found in other blog posts
+    # @attr_reader [String] blog_id the blog ID (Twingly internal identification)
+    # @attr_reader [String] blog_name the name of the blog
+    # @attr_reader [String] blog_url the blog URL
     # @attr_reader [Integer] blog_rank the rank of the blog, based on authority and language.
     #   See https://developer.twingly.com/resources/ranking/#blogrank
-    # @attr_reader [Array] tags
+    # @attr_reader [Integer] authority the blog's authority/influence.
+    #   See https://developer.twingly.com/resources/ranking/#authority
     class Post
-      attr_reader :url, :title, :summary, :language_code, :indexed,
-        :published, :blog_url, :blog_name, :authority, :blog_rank, :tags
+      attr_reader :id, :author, :url, :title, :text, :location_code,
+        :language_code, :coordinates, :links, :tags, :images, :indexed_at,
+        :published_at, :reindexed_at, :inlinks_count, :blog_id, :blog_name,
+        :blog_url, :blog_rank, :authority
 
       # Sets all instance variables for the {Post}, given a Hash.
       #
       # @param [Hash] params containing blog post data.
       def set_values(params)
+        @id            = params.fetch('id')
+        @author        = params.fetch('author')
         @url           = params.fetch('url')
         @title         = params.fetch('title')
-        @summary       = params.fetch('summary')
+        @text          = params.fetch('text')
         @language_code = params.fetch('languageCode')
-        @published     = Time.parse(params.fetch('published'))
-        @indexed       = Time.parse(params.fetch('indexed'))
-        @blog_url      = params.fetch('blogUrl')
+        @location_code = params.fetch('locationCode')
+        @coordinates   = params.fetch('coordinates', {})
+        @links         = params.fetch('links', [])
+        @tags          = params.fetch('tags', [])
+        @images        = params.fetch('images', [])
+        @indexed_at    = Time.parse(params.fetch('indexedAt'))
+        @published_at  = Time.parse(params.fetch('publishedAt'))
+        @reindexed_at  = Time.parse(params.fetch('reindexedAt'))
+        @inlinks_count = params.fetch('inlinksCount').to_i
+        @blog_id       = params.fetch('blogId')
         @blog_name     = params.fetch('blogName')
-        @authority     = params.fetch('authority').to_i
+        @blog_url      = params.fetch('blogUrl')
         @blog_rank     = params.fetch('blogRank').to_i
-        @tags          = params.fetch('tags', [])
+        @authority     = params.fetch('authority').to_i
+      end
+
+      # @deprecated Please use {#text} instead
+      def summary
+        warn "[DEPRECATION] `summary` is deprecated. Please use `text` instead."
+        text
+      end
+
+      # @deprecated Please use {#indexed_at} instead
+      def indexed
+        warn "[DEPRECATION] `indexed` is deprecated. Please use `indexed_at` instead."
+        indexed_at
+      end
+
+      # @deprecated Please use {#published_at} instead
+      def published
+        warn "[DEPRECATION] `published` is deprecated. Please use `published_at` instead."
+        published_at
+      end
+
+      # @deprecated Please use {#links} instead
+      def outlinks
+        warn "[DEPRECATION] `outlinks` is deprecated. Please use `links` instead."
+        links
       end
     end
   end

data/lib/twingly/search/query.rb

@@ -5,11 +5,34 @@ module Twingly
   module Search
     # Twingly Search API query
     #
-    # @attr [String] pattern the search query.
-    # @attr [String] language language to restrict the query to.
+    # @attr [String] search_query the search query.
     # @attr [Client] client the client that this query is connected to.
     class Query
-      attr_accessor :pattern, :language, :client
+      attr_accessor :search_query, :client
+
+      # @deprecated Please use {#search_query} instead
+      def pattern
+        warn "[DEPRECATION] `pattern` is deprecated. Please use `search_query` instead."
+        @search_query
+      end
+
+      # @deprecated Please use {#search_query=} instead
+      def pattern=(search_query)
+        warn "[DEPRECATION] `pattern=` is deprecated. Please use `search_query=` instead."
+        @search_query = search_query
+      end
+
+      # @deprecated Please use {#search_query} instead
+      def language
+        warn "[DEPRECATION] `language` is deprecated. Please use `search_query` instead."
+        @language
+      end
+
+      # @deprecated Please use {#search_query=} instead
+      def language=(language_code)
+        warn "[DEPRECATION] `language=` is deprecated. Please use `search_query=` instead."
+        @language = language_code
+      end
 
       # @return [Time] the time that was set with {#start_time=}.
       def start_time
@@ -36,7 +59,7 @@ module Twingly
 
       # Executes the query and returns the result.
       #
-      # @raise [QueryError] if {#pattern} is empty.
+      # @raise [QueryError] if {#search_query} is empty.
       # @raise [AuthError] if the API couldn't authenticate you. Make sure your API key is correct.
       # @raise [ServerError] if the query could not be executed due to a server error.
       # @return [Result] the result for this query.
@@ -50,18 +73,21 @@ module Twingly
         Faraday::Utils.build_query(request_parameters)
       end
 
-      # @raise [QueryError] if {#pattern} is empty.
+      # @raise [QueryError] if {#search_query} is empty.
       # @return [Hash] the request parameters.
       def request_parameters
-        fail QueryError, "Missing pattern" if pattern.to_s.empty?
+        full_search_query = search_query.to_s.dup
+        full_search_query << " lang:#{@language}" unless @language.to_s.empty?
+        full_search_query << " start-date:#{formatted_start_date}" if start_time
+        full_search_query << " end-date:#{formatted_end_date}"     if end_time
+
+        if full_search_query.to_s.empty?
+          fail QueryError, "Search query cannot be empty"
+        end
 
         {
-          key: client.api_key,
-          searchpattern: pattern,
-          documentlang: language,
-          ts: ts,
-          tsTo: ts_to,
-          xmloutputversion: 2,
+          apikey: client.api_key,
+          q: full_search_query
         }
       end
 
@@ -95,12 +121,16 @@ module Twingly
         fail QueryError, "Not a Time object" unless time.respond_to?(:to_time)
       end
 
-      def ts
-        start_time.to_time.utc.strftime("%F %T") if start_time
+      def formatted_start_date
+        format_timestamp_for_query(start_time) if start_time
+      end
+
+      def formatted_end_date
+        format_timestamp_for_query(end_time) if end_time
       end
 
-      def ts_to
-        end_time.to_time.utc.strftime("%F %T") if end_time
+      def format_timestamp_for_query(timestamp)
+        timestamp.to_time.utc.strftime("%FT%T")
       end
     end
   end

data/lib/twingly/search/result.rb

@@ -12,6 +12,16 @@ module Twingly
     class Result
       attr_accessor :number_of_matches_returned, :number_of_matches_total,
         :seconds_elapsed
+      attr_writer :incomplete_result
+
+      # @return [true] if one or multiple servers were too slow to respond
+      #   within the maximum allowed query time.
+      # @return [false] if all servers responded within the maximum allowed
+      #   query time.
+      # @see https://developer.twingly.com/resources/search/#response
+      def incomplete?
+        @incomplete_result
+      end
 
       # @return [Array<Post>] all posts that matched the {Query}.
       def posts
@@ -28,6 +38,7 @@ module Twingly
         matches = "@posts, "
         matches << "@number_of_matches_returned=#{self.number_of_matches_returned}, "
         matches << "@number_of_matches_total=#{self.number_of_matches_total}"
+        matches << "@incomplete_result=#{self.incomplete?}"
 
         sprintf("#<%s:0x%x %s>", self.class.name, __id__, matches)
       end

data/lib/twingly/search/version.rb

@@ -1,5 +1,5 @@
 module Twingly
   module Search
-    VERSION = "5.0.1"
+    VERSION = "5.1.0"
   end
 end

data/spec/client_spec.rb

@@ -90,13 +90,13 @@ describe Client do
 
       let(:query) do
         query = subject.query
-        query.pattern = "something"
+        query.search_query = "something"
         query
       end
 
       it "should raise error on invalid API key" do
         VCR.use_cassette("search_without_valid_api_key") do
-          expect { subject.execute_query(query) }.to raise_error(AuthError, "The API key does not exist.")
+          expect { subject.execute_query(query) }.to raise_error(AuthError, /Unauthorized/)
         end
       end
     end

data/spec/error_spec.rb

@@ -3,17 +3,37 @@ require "spec_helper"
 describe Twingly::Search::Error do
   it { is_expected.to be_a(StandardError) }
 
-  describe ".from_api_response_message" do
-    subject { described_class.from_api_response_message(server_response_message) }
+  let(:message) { "This is the error message!" }
 
-    context "when given message containing 'API key'" do
-      let(:server_response_message) { "... API key ..." }
+  describe ".from_api_response" do
+    subject { described_class.from_api_response(code, message) }
 
-      it { is_expected.to be_an(AuthError) }
+    context "when given code 401" do
+      let(:code) { 401 }
+
+      it { is_expected.to be_a(AuthError) }
+    end
+
+    context "when given code 402" do
+      let(:code) { 402 }
+
+      it { is_expected.to be_a(AuthError) }
+    end
+
+    context "when given code 400" do
+      let(:code) { 400 }
+
+      it { is_expected.to be_a(QueryError) }
+    end
+
+    context "when given code 404" do
+      let(:code) { 404 }
+
+      it { is_expected.to be_a(QueryError) }
     end
 
-    context "when given a server error message" do
-      let(:server_response_message) { "An error occured." }
+    context "when given another code" do
+      let(:code) { 500 }
 
       it { is_expected.to be_a(ServerError) }
     end

data/spec/fixtures/incomplete_result.xml

@@ -0,0 +1,2 @@
+<?xml version="1.0" encoding="utf-8"?>
+<twinglydata numberOfMatchesReturned="0" secondsElapsed="0.203" numberOfMatchesTotal="0" incompleteResult="true" />

data/spec/fixtures/livefeed/empty_api_key_result.xml

@@ -0,0 +1,3 @@
+<error code="40001">
+<message>Parameter apikey may not be empty</message>
+</error>

data/spec/fixtures/livefeed/non_xml_result.xml

@@ -0,0 +1 @@
+<html><body>This is a non XML fixture.</body></html>

data/spec/fixtures/livefeed/not_found_result.xml

@@ -0,0 +1,3 @@
+<error code="40401">
+<message>Not Found</message>
+</error>

data/spec/fixtures/livefeed/service_unavailable_result.xml

@@ -0,0 +1,3 @@
+<error code="50301">
+<message>Authentication service unavailable</message>
+</error>

data/spec/fixtures/livefeed/unauthorized_api_key_result.xml

@@ -0,0 +1,3 @@
+<error code="40101">
+<message>Unauthorized</message>
+</error>

data/spec/fixtures/livefeed/valid_empty_result.xml

@@ -0,0 +1,2 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<twinglydata ts="2017-04-25T08:25:35.9747845Z" from="2017-04-25T22:00:00Z" numberOfPosts="0" maxNumberOfPosts="3" nextTimestamp="2017-04-25T22:00:00Z"/>

data/spec/fixtures/livefeed/valid_result.xml

@@ -0,0 +1,79 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<twinglydata ts="2017-04-11T15:09:48.8750635Z" from="2017-04-10T22:00:00Z" numberOfPosts="3" maxNumberOfPosts="3" firstPost="2017-04-10T22:00:29.267Z" lastPost="2017-04-10T22:11:47.243Z" nextTimestamp="2017-04-10T22:11:47.244Z">
+  <post>
+    <id>727444183574244541</id>
+    <author />
+    <url>http://flinnman.blogg.se/2017/april/mandag-igen.html</url>
+    <title>Måndag igen</title>
+    <text>Hoppla hejsan bloggy!Måndag. Förutom trist hosta på E så mår vi fint. Vi använde helgen till massa umgänge med nära och kära. Kändes extra skönt efter fredagen. Vi hade en tyst minut i en full matsal i polishuset idag. Det var fint! Well. Ny vecka och ... - Läs hela inlägget här</text>
+    <languageCode>sv</languageCode>
+    <locationCode>se</locationCode>
+    <coordinates />
+    <links />
+    <tags>
+      <tag>Jag</tag>
+    </tags>
+    <images />
+    <indexedAt>2017-04-10T22:00:24Z</indexedAt>
+    <publishedAt>2017-04-10T19:11:11Z</publishedAt>
+    <reindexedAt>2017-04-10T22:00:24Z</reindexedAt>
+    <inlinksCount>0</inlinksCount>
+    <blogId>10357806725947705095</blogId>
+    <blogName>Frida L</blogName>
+    <blogUrl>http://flinnman.blogg.se</blogUrl>
+    <blogRank>1</blogRank>
+    <authority>2</authority>
+  </post>
+  <post>
+    <id>6564050082079070812</id>
+    <author />
+    <url>http://malinbs.blogg.se/2017/april/du-ska-hedra-din-fader-och-din-moder.html</url>
+    <title>Du ska hedra din fader och din moder</title>
+    <text>Vilken solig och god helg det har varit på mina breddgrader! Minns för en massa år sedan (25? Eller lite mera?) när Herrn i Huset byggde altanen och samtidigt såg till att det blev eluttag där. Min mammas första kommentar blev: Tänk så bra med ett uttag, ... - Läs hela inlägget här</text>
+    <languageCode>sv</languageCode>
+    <locationCode>se</locationCode>
+    <coordinates />
+    <links />
+    <tags>
+      <tag>Allmänt</tag>
+    </tags>
+    <images />
+    <indexedAt>2017-04-10T22:00:16Z</indexedAt>
+    <publishedAt>2017-04-09T17:12:08Z</publishedAt>
+    <reindexedAt>2017-04-10T22:00:16Z</reindexedAt>
+    <inlinksCount>0</inlinksCount>
+    <blogId>14781290076709326355</blogId>
+    <blogName>Blogga, ett sätt att umgås!</blogName>
+    <blogUrl>http://malinbs.blogg.se</blogUrl>
+    <blogRank>1</blogRank>
+    <authority>4</authority>
+  </post>
+  <post>
+    <id>3062976931264108164</id>
+    <author>josegacel</author>
+    <url>https://josegabrielcelis.wordpress.com/2017/04/09/1476/</url>
+    <title />
+    <text>from Instagram: http://ift.tt/2ofZdhV</text>
+    <languageCode>sv</languageCode>
+    <locationCode />
+    <coordinates />
+    <links>
+      <link>http://www.ift.tt/2ofZdhV</link>
+      <link>http://feeds.wordpress.com/1.0/gocomments/josegabrielcelis.wordpress.com/1476</link>
+    </links>
+    <tags>
+      <tag>Fotos</tag>
+      <tag>Instagram</tag>
+    </tags>
+    <images />
+    <indexedAt>2017-04-10T22:00:28Z</indexedAt>
+    <publishedAt>2017-04-09T22:31:34Z</publishedAt>
+    <reindexedAt>2017-04-10T22:00:28Z</reindexedAt>
+    <inlinksCount>0</inlinksCount>
+    <blogId>1811310581070495497</blogId>
+    <blogName>José Gabriel Celis</blogName>
+    <blogUrl>https://josegabrielcelis.wordpress.com</blogUrl>
+    <blogRank>1</blogRank>
+    <authority>0</authority>
+  </post>
+</twinglydata>